From 7ef9747fb936b010f4931c98d395fc1ef51b3797 Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Mon, 24 Oct 2022 15:58:14 -0500
Subject: [PATCH 001/144] add environment file

---
 environment.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 environment.yml

diff --git a/environment.yml b/environment.yml
new file mode 100644
index 00000000..684aad1f
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,20 @@
+name: asf-stac
+channels:
+  - hyp3
+  - conda-forge
+  - nodefaults
+dependencies:
+  - boto3
+  - botocore
+  - python
+  - pip
+  # For packaging, and testing
+  - flake8
+  - flake8-import-order
+  - flake8-blind-except
+  - flake8-builtins
+  - pytest
+  - pytest-console-scripts
+  - pytest-cov
+  # For running
+  - pystac

From a2b9d56808bfdcc055fc2c73173e8c44b39e2935 Mon Sep 17 00:00:00 2001
From: Forrest Williams <31411324+forrestfwilliams@users.noreply.github.com>
Date: Mon, 24 Oct 2022 16:27:11 -0500
Subject: [PATCH 002/144] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 19db3705..1c486c21 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,3 @@
-# ASF_STAC
+# asf-stac
 
 A repository containing code related to the creation and hosting of STAC catalogs by the ASF tools team.

From 58c972bfd45e1a3782f12f3b671ceb1cece2abe5 Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Tue, 25 Oct 2022 10:13:46 -0500
Subject: [PATCH 003/144] initialize coherence stac script, add filename
 parsing

---
 environment.yml       |  1 +
 src/coherence_stac.py | 84 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 85 insertions(+)
 create mode 100644 src/coherence_stac.py

diff --git a/environment.yml b/environment.yml
index 684aad1f..da874a51 100644
--- a/environment.yml
+++ b/environment.yml
@@ -18,3 +18,4 @@ dependencies:
   - pytest-cov
   # For running
   - pystac
+  - shapely
diff --git a/src/coherence_stac.py b/src/coherence_stac.py
new file mode 100644
index 00000000..6cd3d925
--- /dev/null
+++ b/src/coherence_stac.py
@@ -0,0 +1,84 @@
+from datetime import datetime
+from pathlib import Path
+
+import boto3
+# import pystac
+from shapely import geometry
+
+'''
+Design:
+
+[x] connect to aws open bucket containing coherence data
+[x] ls the prefixes we're interested in
+[x] parse names (N00E014_007D_inc.tif) -> bbox, season, polarization, dataset
+[ ] create base sentinel-1 stac item using https://github.com/stac-extensions/sar
+[ ] add coherence metadata to base
+[ ] add to catalog
+[ ] save catalog locally
+
+structure:
+    item (tile + season)
+        inc
+        lsmap
+        COH(all 4)
+        rho
+        rmse
+        tau
+'''
+
+SEASONS = {
+    'winter': (datetime(2019, 12, 1), datetime(2020, 2, 28)),
+    'spring': (datetime(2020, 3, 1), datetime(2020, 5, 31)),
+    'summer': (datetime(2020, 6, 1), datetime(2020, 8, 31)),
+    'fall': (datetime(2020, 9, 1), datetime(2020, 11, 30)),
+}
+
+
+def construct_urls(s3_client, bucket, keys):
+    location = s3_client.get_bucket_location(Bucket=bucket)['LocationConstraint']
+    urls = [f'https://{bucket}.s3.{location}.amazonaws.com/{k}' for k in keys]
+    return urls
+
+
+def get_object_urls(s3_client, bucket, prefix, requester_pays=False):
+    kwargs = {'RequestPayer': 'requester'} if requester_pays else {}
+    response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix, **kwargs)
+    keys = [x['Key'] for x in response['Contents']]
+    urls = construct_urls(s3_client, bucket, keys)
+    return urls
+
+
+# TODO why is there an extra zero in N48W090, will this cause issues?
+def tileid_to_bbox(tileid):
+    north = int(tileid[1:3])
+    if tileid[0] == 'S':
+        north *= -1
+    south = north - 1
+
+    west = int(tileid[4:7])
+    if tileid[3] == 'W':
+        west *= -1
+    east = west + 1
+    bbox = geometry.box(west, south, east, north)
+    return bbox
+
+
+def parse_url(url):
+    parts = Path(url).stem.split('_')
+    if len(parts) == 3:
+        tileid, orbit, product = parts
+        bbox = tileid_to_bbox(tileid)
+        return {'url': url, 'bbox': bbox, 'product': product}
+
+    tileid, season, polarization, product = parts
+    bbox = tileid_to_bbox(tileid)
+    date_range = SEASONS[season]
+    return {'url': url, 'bbox': bbox, 'product': product, 'date_range': date_range, 'polarization': polarization}
+
+
+if __name__ == '__main__':
+    bucket = 'sentinel-1-global-coherence-earthbigdata'
+    prefix = 'data/tiles/N48W005/'
+    s3 = boto3.client('s3')
+    urls = get_object_urls(s3, bucket, prefix, requester_pays=True)
+    asset_dicts = [parse_url[x] for x in urls]

From 41f6692d9f83443d7efb1b1a5425ba13699d30db Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Tue, 25 Oct 2022 14:03:27 -0500
Subject: [PATCH 004/144] first working version

---
 src/coherence_stac.py | 101 ++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 97 insertions(+), 4 deletions(-)

diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index 6cd3d925..1522e81a 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -2,7 +2,8 @@
 from pathlib import Path
 
 import boto3
-# import pystac
+import pystac
+from pystac.extensions import sar
 from shapely import geometry
 
 '''
@@ -26,6 +27,7 @@
         tau
 '''
 
+SENTINEL1_CENTER_FREQUENCY = 5.405
 SEASONS = {
     'winter': (datetime(2019, 12, 1), datetime(2020, 2, 28)),
     'spring': (datetime(2020, 3, 1), datetime(2020, 5, 31)),
@@ -68,17 +70,108 @@ def parse_url(url):
     if len(parts) == 3:
         tileid, orbit, product = parts
         bbox = tileid_to_bbox(tileid)
-        return {'url': url, 'bbox': bbox, 'product': product}
+        metadata = {'url': url, 'bbox': bbox, tileid: tileid, 'product': product}
+        return metadata
 
     tileid, season, polarization, product = parts
     bbox = tileid_to_bbox(tileid)
     date_range = SEASONS[season]
-    return {'url': url, 'bbox': bbox, 'product': product, 'date_range': date_range, 'polarization': polarization}
+    metadata = {
+        'url': url,
+        'bbox': bbox,
+        'tileid': tileid,
+        'product': product,
+        'date_range': date_range,
+        'season': season,
+        'polarization': polarization,
+    }
+    return metadata
+
+
+def create_stac_item(yearly_assets, seasonal_assets):
+    ex_asset = seasonal_assets[0]
+    item_id = f'{ex_asset["tileid"]}_{ex_asset["season"]}'
+    properties = {'tileid': ex_asset['tileid'], 'season': ex_asset['season']}
+
+    item = pystac.Item(
+        id=item_id,
+        geometry=geometry.mapping(ex_asset['bbox']),
+        bbox=ex_asset['bbox'].bounds,
+        datetime=ex_asset['date_range'][0],
+        properties=properties,
+    )
+
+    # TODO don't know if look properties are correct
+    ext_sar = sar.SarExtension.ext(item, add_if_missing=True)
+    ext_sar.apply(
+        'IW',
+        sar.FrequencyBand('C'),
+        [sar.Polarization('VV'), sar.Polarization('VH')],
+        'COH',
+        SENTINEL1_CENTER_FREQUENCY,
+        looks_range=7,
+        looks_azimuth=12,
+        observation_direction=sar.ObservationDirection('right'),
+    )
+
+    for asset in yearly_assets:
+        item.add_asset(
+            key=asset['product'],
+            asset=pystac.Asset(href=asset['url'], media_type=pystac.MediaType.GEOTIFF),
+        )
+
+    for asset in seasonal_assets:
+        key = f'{asset["product"]}_{asset["polarization"]}'
+        asset_properties = {'polarization': asset['polarization']}
+        if 'COH' in asset['product']:
+            asset_properties['product'] = 'COH'
+            asset_properties['temporal_separation'] = f'{asset["product"][-2:]} days'
+        else:
+            asset_properties['product'] = asset['product']
+
+        item.add_asset(
+            key=key,
+            asset=pystac.Asset(href=asset['url'], media_type=pystac.MediaType.GEOTIFF, extra_fields=asset_properties),
+        )
+    return item
+
+
+def create_stac_catalog(items):
+    extension_list = [x.to_dict()['stac_extensions'] for x in items]
+    extensions = list(set([num for sublist in extension_list for num in sublist]))
+    catalog = pystac.Catalog(
+        id='sentinel-1-global-coherence-earthbigdata',
+        description='A catalog containing the Earthbigdata Sentinel-1 Global Coherence Dataset',
+        catalog_type=pystac.CatalogType.ABSOLUTE_PUBLISHED,
+        stac_extensions=extensions
+    )
+    catalog.add_items(items)
+    return catalog
+
+
+def save_stac_catalog_locally(catalog, catalog_name: str):
+    catalog_name = Path(catalog_name)
+    if not catalog_name.exists():
+        catalog_name.mkdir()
+    catalog.normalize_hrefs(str(catalog_name))
+    catalog.save()
+    return catalog_name / 'catalog.json'
 
 
 if __name__ == '__main__':
     bucket = 'sentinel-1-global-coherence-earthbigdata'
     prefix = 'data/tiles/N48W005/'
     s3 = boto3.client('s3')
+
     urls = get_object_urls(s3, bucket, prefix, requester_pays=True)
-    asset_dicts = [parse_url[x] for x in urls]
+    asset_dicts = [parse_url(x) for x in urls]
+    items = []
+
+    yearly_assets = [x for x in asset_dicts if ('inc' in x['url']) or ('lsmap' in x['url'])]
+    for season in ('spring', 'summer', 'fall', 'winter'):
+        seasonal_assets = [x for x in asset_dicts if season in x['url']]
+        item = create_stac_item(yearly_assets, seasonal_assets)
+        items.append(item)
+
+    catalog = create_stac_catalog(items)
+    save_stac_catalog_locally(catalog, 'coherence_catalog')

From a18eb77d5511088d2c54cb7a88bbca7dfc1f7f0f Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Tue, 25 Oct 2022 14:04:04 -0500
Subject: [PATCH 005/144] add france url list

---
 data/france_urls.txt | 150 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 150 insertions(+)
 create mode 100644 data/france_urls.txt

diff --git a/data/france_urls.txt b/data/france_urls.txt
new file mode 100644
index 00000000..eb1c9bd9
--- /dev/null
+++ b/data/france_urls.txt
@@ -0,0 +1,150 @@
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E000/N42E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E001/N42E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E002/N42E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E003/N42E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E004/N42E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E005/N42E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E006/N42E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E007/N42E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E008/N42E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42E009/N42E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42W001/N42W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42W002/N42W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42W003/N42W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42W004/N42W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N42W005/N42W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E000/N43E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E001/N43E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E002/N43E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E003/N43E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E004/N43E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E005/N43E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E006/N43E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E007/N43E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E008/N43E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43E009/N43E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43W001/N43W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43W002/N43W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43W003/N43W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43W004/N43W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N43W005/N43W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E000/N44E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E001/N44E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E002/N44E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E003/N44E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E004/N44E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E005/N44E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E006/N44E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E007/N44E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E008/N44E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44E009/N44E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44W001/N44W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44W002/N44W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44W003/N44W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44W004/N44W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N44W005/N44W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E000/N45E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E001/N45E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E002/N45E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E003/N45E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E004/N45E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E005/N45E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E006/N45E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E007/N45E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E008/N45E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45E009/N45E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45W001/N45W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45W002/N45W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45W003/N45W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45W004/N45W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N45W005/N45W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E000/N46E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E001/N46E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E002/N46E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E003/N46E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E004/N46E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E005/N46E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E006/N46E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E007/N46E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E008/N46E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46E009/N46E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46W001/N46W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46W002/N46W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46W003/N46W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46W004/N46W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N46W005/N46W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E000/N47E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E001/N47E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E002/N47E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E003/N47E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E004/N47E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E005/N47E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E006/N47E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E007/N47E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E008/N47E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47E009/N47E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47W001/N47W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47W002/N47W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47W003/N47W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47W004/N47W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N47W005/N47W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E000/N48E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E001/N48E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E002/N48E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E003/N48E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E004/N48E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E005/N48E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E006/N48E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E007/N48E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E008/N48E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48E009/N48E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48W001/N48W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48W002/N48W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48W003/N48W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48W004/N48W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N48W005/N48W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E000/N49E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E001/N49E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E002/N49E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E003/N49E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E004/N49E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E005/N49E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E006/N49E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E007/N49E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E008/N49E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49E009/N49E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49W001/N49W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49W002/N49W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49W003/N49W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49W004/N49W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N49W005/N49W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E000/N50E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E001/N50E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E002/N50E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E003/N50E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E004/N50E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E005/N50E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E006/N50E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E007/N50E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E008/N50E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50E009/N50E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50W001/N50W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50W002/N50W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50W003/N50W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50W004/N50W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N50W005/N50W005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E000/N51E000_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E001/N51E001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E002/N51E002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E003/N51E003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E004/N51E004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E005/N51E005_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E006/N51E006_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E007/N51E007_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E008/N51E008_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51E009/N51E009_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51W001/N51W001_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51W002/N51W002_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51W003/N51W003_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51W004/N51W004_summer_vv_COH12.tif
+https://sentinel-1-global-coherence-earthbigdata.s3.us-west-2.amazonaws.com/data/tiles/N51W005/N51W005_summer_vv_COH12.tif

From ba16317baac2a958194705af23d97e5ba90f828a Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Tue, 25 Oct 2022 14:06:04 -0500
Subject: [PATCH 006/144] update design checklist

---
 src/coherence_stac.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index 1522e81a..9630a9a1 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -12,10 +12,10 @@
 [x] connect to aws open bucket containing coherence data
 [x] ls the prefixes we're interested in
 [x] parse names (N00E014_007D_inc.tif) -> bbox, season, polarization, dataset
-[ ] create base sentinel-1 stac item using https://github.com/stac-extensions/sar
-[ ] add coherence metadata to base
-[ ] add to catalog
-[ ] save catalog locally
+[x] create base sentinel-1 stac item using https://github.com/stac-extensions/sar
+[x] add coherence metadata to base
+[x] add to catalog
+[x] save catalog locally
 
 structure:
     item (tile + season)

From a5589c78b4c68409f6e609920b4a9bc493db2e4b Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Thu, 27 Oct 2022 10:33:06 -0500
Subject: [PATCH 007/144] switch to using tile-based collections

---
 src/coherence_stac.py | 106 +++++++++++++++++++++++++++++-------------
 1 file changed, 74 insertions(+), 32 deletions(-)

diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index 9630a9a1..a9678cd2 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -5,6 +5,7 @@
 import pystac
 from pystac.extensions import sar
 from shapely import geometry
+from tqdm import tqdm
 
 '''
 Design:
@@ -29,24 +30,24 @@
 
 SENTINEL1_CENTER_FREQUENCY = 5.405
 SEASONS = {
-    'winter': (datetime(2019, 12, 1), datetime(2020, 2, 28)),
-    'spring': (datetime(2020, 3, 1), datetime(2020, 5, 31)),
-    'summer': (datetime(2020, 6, 1), datetime(2020, 8, 31)),
-    'fall': (datetime(2020, 9, 1), datetime(2020, 11, 30)),
+    'WINTER': (datetime(2019, 12, 1), datetime(2020, 2, 28)),
+    'SPRING': (datetime(2020, 3, 1), datetime(2020, 5, 31)),
+    'SUMMER': (datetime(2020, 6, 1), datetime(2020, 8, 31)),
+    'FALL': (datetime(2020, 9, 1), datetime(2020, 11, 30)),
 }
 
 
-def construct_urls(s3_client, bucket, keys):
+def construct_url(s3_client, bucket, key):
     location = s3_client.get_bucket_location(Bucket=bucket)['LocationConstraint']
-    urls = [f'https://{bucket}.s3.{location}.amazonaws.com/{k}' for k in keys]
-    return urls
+    url = f'https://{bucket}.s3.{location}.amazonaws.com/{key}'
+    return url
 
 
 def get_object_urls(s3_client, bucket, prefix, requester_pays=False):
     kwargs = {'RequestPayer': 'requester'} if requester_pays else {}
     response = s3_client.list_objects_v2(Bucket=bucket, Prefix=prefix, **kwargs)
     keys = [x['Key'] for x in response['Contents']]
-    urls = construct_urls(s3_client, bucket, keys)
+    urls = [construct_url(s3_client, bucket, x) for x in keys]
     return urls
 
 
@@ -66,11 +67,11 @@ def tileid_to_bbox(tileid):
 
 
 def parse_url(url):
-    parts = Path(url).stem.split('_')
+    parts = Path(url.upper()).stem.split('_')
     if len(parts) == 3:
         tileid, orbit, product = parts
         bbox = tileid_to_bbox(tileid)
-        metadata = {'url': url, 'bbox': bbox, tileid: tileid, 'product': product}
+        metadata = {'url': url, 'bbox': bbox, 'tileid': tileid, 'product': product}
         return metadata
 
     tileid, season, polarization, product = parts
@@ -91,13 +92,20 @@ def parse_url(url):
 def create_stac_item(yearly_assets, seasonal_assets):
     ex_asset = seasonal_assets[0]
     item_id = f'{ex_asset["tileid"]}_{ex_asset["season"]}'
-    properties = {'tileid': ex_asset['tileid'], 'season': ex_asset['season']}
-
+    start_date = ex_asset['date_range'][0]
+    end_date = ex_asset['date_range'][1]
+    mid_date = start_date + (end_date - start_date) / 2
+    properties = {
+        'tileid': ex_asset['tileid'],
+        'season': ex_asset['season'],
+        'start_datetime': start_date.isoformat(),
+        'end_datetime': end_date.isoformat(),
+    }
     item = pystac.Item(
         id=item_id,
         geometry=geometry.mapping(ex_asset['bbox']),
         bbox=ex_asset['bbox'].bounds,
-        datetime=ex_asset['date_range'][0],
+        datetime=mid_date,
         properties=properties,
     )
 
@@ -125,7 +133,7 @@ def create_stac_item(yearly_assets, seasonal_assets):
         asset_properties = {'polarization': asset['polarization']}
         if 'COH' in asset['product']:
             asset_properties['product'] = 'COH'
-            asset_properties['temporal_separation'] = f'{asset["product"][-2:]} days'
+            asset_properties['temporal_separation'] = f'{int(asset["product"][-2:])} days'
         else:
             asset_properties['product'] = asset['product']
 
@@ -136,16 +144,41 @@ def create_stac_item(yearly_assets, seasonal_assets):
     return item
 
 
-def create_stac_catalog(items):
-    extension_list = [x.to_dict()['stac_extensions'] for x in items]
-    extensions = list(set([num for sublist in extension_list for num in sublist]))
+def create_tile_stac_collection(
+    s3_client, bucket, prefix, date_interval=(datetime(2019, 12, 1), datetime(2020, 11, 30))
+):
+    urls = get_object_urls(s3_client, bucket, prefix, requester_pays=True)
+    asset_dicts = [parse_url(x) for x in urls]
+    items = []
+
+    yearly_assets = [x for x in asset_dicts if ('inc' in x['url']) or ('lsmap' in x['url'])]
+    for season in ('spring', 'summer', 'fall', 'winter'):
+        seasonal_assets = [x for x in asset_dicts if season in x['url']]
+        item = create_stac_item(yearly_assets, seasonal_assets)
+        items.append(item)
+
+    tileid = asset_dicts[0]['tileid']
+    spatial_extent = pystac.SpatialExtent(items[0].bbox)
+    temporal_extent = pystac.TemporalExtent(intervals=[date_interval])
+    collection_extent = pystac.Extent(spatial=spatial_extent, temporal=temporal_extent)
+    collection = pystac.Collection(
+        id=tileid,
+        description=f'Sentinel-1 Coherence Tile {tileid}',
+        extent=collection_extent,
+    )
+    collection.add_items(items)
+    return collection
+
+
+def create_stac_catalog():
+    # extension_list = [x.to_dict()['stac_extensions'] for x in items]
+    # extensions = list(set([num for sublist in extension_list for num in sublist]))
     catalog = pystac.Catalog(
         id='sentinel-1-global-coherence-earthbigdata',
         description='A catalog containing the Earthbigdata Sentinel-1 Global Coherence Dataset',
-        catalog_type=pystac.CatalogType.ABSOLUTE_PUBLISHED,
-        stac_extensions=extensions
+        catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED,
+        # stac_extensions=extensions,
     )
-    catalog.add_items(items)
     return catalog
 
 
@@ -158,20 +191,29 @@ def save_stac_catalog_locally(catalog, catalog_name: str):
     return catalog_name / 'catalog.json'
 
 
+def save_stac_catalog_s3(catalog, s3_client, bucket, key):
+    base_url = Path(construct_url(s3_client, bucket, key))
+    catalog_name = base_url.name
+    catalog.normalize_hrefs(str(base_url))
+    catalog.save(dest_href=catalog_name)
+    # jsons = [x for x in Path(catalog_name).glob('**/*json')]
+    # print('uploading...')
+    # for json in tqdm(jsons):
+    #     s3_client.upload_file(str(json), bucket, str(Path(key).parent / json))
+    return f'{catalog_name}/catalog.json'
+
+
 if __name__ == '__main__':
     bucket = 'sentinel-1-global-coherence-earthbigdata'
-    prefix = 'data/tiles/N48W005/'
+    tiles = ['N48W005', 'N49W005']
     s3 = boto3.client('s3')
 
-    urls = get_object_urls(s3, bucket, prefix, requester_pays=True)
-    asset_dicts = [parse_url(x) for x in urls]
-    items = []
-
-    yearly_assets = [x for x in asset_dicts if ('inc' in x['url']) or ('lsmap' in x['url'])]
-    for season in ('spring', 'summer', 'fall', 'winter'):
-        seasonal_assets = [x for x in asset_dicts if season in x['url']]
-        item = create_stac_item(yearly_assets, seasonal_assets)
-        items.append(item)
+    catalog = create_stac_catalog()
+    for tile in tiles:
+        prefix = f'data/tiles/{tile}/'
+        collection = create_tile_stac_collection(s3, bucket, prefix)
+        catalog.add_child(collection)
 
-    catalog = create_stac_catalog(items)
-    save_stac_catalog_locally(catalog, 'coherence_catalog')
+    upload_bucket = 'ffwilliams2-shenanigans'
+    upload_key = 'stac/coherence_stac'
+    save_stac_catalog_s3(catalog, s3, upload_bucket, upload_key)

From efa6270c0a9293e3d37f23ae06b470a06c3b6c97 Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Thu, 27 Oct 2022 11:08:22 -0500
Subject: [PATCH 008/144] add descriptions and prep for france processing

---
 environment.yml       |  6 ++---
 src/coherence_stac.py | 54 +++++++++++++++++++++++++++++++++++--------
 2 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/environment.yml b/environment.yml
index da874a51..b9c8862b 100644
--- a/environment.yml
+++ b/environment.yml
@@ -8,6 +8,9 @@ dependencies:
   - botocore
   - python
   - pip
+  - pystac
+  - shapely
+  - tqdm
   # For packaging, and testing
   - flake8
   - flake8-import-order
@@ -16,6 +19,3 @@ dependencies:
   - pytest
   - pytest-console-scripts
   - pytest-cov
-  # For running
-  - pystac
-  - shapely
diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index a9678cd2..73c03689 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -35,6 +35,34 @@
     'SUMMER': (datetime(2020, 6, 1), datetime(2020, 8, 31)),
     'FALL': (datetime(2020, 9, 1), datetime(2020, 11, 30)),
 }
+LICENSE = 'Creative Commons Attribution 4.0 International Public License'
+
+DATA_CITATION = (
+    'Kellndorfer, J. , O. Cartus, M. Lavalle,  C. Magnard, P. Milillo, S. Oveisgharan, B. Osmanoglu, '
+    'P. Rosen, and U. Wegmuller. 2022. Global seasonal Sentinel-1 interferometric coherence and backscatter data set. '
+    '[Indicate subset used]. Fairbanks, Alaska USA. NASA Alaska Satellite Facility Synthetic Aperture Radar '
+    'Distributed Active Archive Center. doi: https://doi.org/10.5067/8W33RRS6S2RV. [Date Accessed].'
+)
+
+LITERATURE_CITATION = (
+    'Kellndorfer, J. , O. Cartus, M. Lavalle,  C. Magnard, P. Milillo, S. Oveisgharan, B. '
+    'Osmanoglu, P. Rosen, and U. Wegmuller. 2022. Global seasonal Sentinel-1 interferometric coherence and '
+    'backscatter data set., Scientific Data. https://doi.org/10.1038/s41597-022-01189-6'
+)
+
+DESCRIPTION = (
+    'This data set is the first-of-its-kind spatial representation of multi-seasonal, global C-band '
+    'Synthetic Aperture Radar (SAR) interferometric repeat-pass coherence and backscatter signatures. Coverage '
+    'comprises land masses and ice sheets from 82° Northern to 79° Southern latitudes. The data set is derived from '
+    'multi-temporal repeat-pass interferometric processing of about 205,000 Sentinel-1 C-band SAR images acquired in '
+    'Interferometric Wide-Swath Mode from 1-Dec-2019 to 30-Nov-2020. The data set encompasses three sets of seasonal '
+    '(December-February, March-May, June-August, September-November) metrics produced with a pixel spacing of three '
+    'arcseconds: 1) Median 6-, 12-, 18-, 24-, 36-, and 48-days repeat-pass coherence at VV or HH polarizations, 2) '
+    'Mean radiometrically terrain corrected backscatter (γ0) at VV and VH, or HH and HV polarizations, and 3) '
+    'Estimated parameters of an exponential coherence decay model. The data set has been produced to obtain global, '
+    'spatially detailed information on how decorrelation affects interferometric measurements of surface displacement '
+    'and is rich in spatial and temporal information for a variety of mapping applications.'
+)
 
 
 def construct_url(s3_client, bucket, key):
@@ -171,13 +199,12 @@ def create_tile_stac_collection(
 
 
 def create_stac_catalog():
-    # extension_list = [x.to_dict()['stac_extensions'] for x in items]
-    # extensions = list(set([num for sublist in extension_list for num in sublist]))
+    extra_fields = {'License': LICENSE, 'Data Citation': DATA_CITATION, 'Literature Citation': LITERATURE_CITATION}
     catalog = pystac.Catalog(
         id='sentinel-1-global-coherence-earthbigdata',
-        description='A catalog containing the Earthbigdata Sentinel-1 Global Coherence Dataset',
+        description=DESCRIPTION,
         catalog_type=pystac.CatalogType.RELATIVE_PUBLISHED,
-        # stac_extensions=extensions,
+        extra_fields=extra_fields,
     )
     return catalog
 
@@ -196,16 +223,25 @@ def save_stac_catalog_s3(catalog, s3_client, bucket, key):
     catalog_name = base_url.name
     catalog.normalize_hrefs(str(base_url))
     catalog.save(dest_href=catalog_name)
-    # jsons = [x for x in Path(catalog_name).glob('**/*json')]
-    # print('uploading...')
-    # for json in tqdm(jsons):
-    #     s3_client.upload_file(str(json), bucket, str(Path(key).parent / json))
+    jsons = [x for x in Path(catalog_name).glob('**/*json')]
+    print('uploading...')
+    for json in tqdm(jsons):
+        s3_client.upload_file(str(json), bucket, str(Path(key).parent / json))
     return f'{catalog_name}/catalog.json'
 
 
+def parse_france_list(data_path):
+    breakpoint()
+    with open(data_path, 'r') as f:
+        urls = [x.strip() for x in f.readlines()]
+    tileids = [parse_url(x)['tileid'] for x in urls]
+    return list(set(tileids))
+
+
 if __name__ == '__main__':
     bucket = 'sentinel-1-global-coherence-earthbigdata'
-    tiles = ['N48W005', 'N49W005']
+    # tiles = ['N48W005', 'N49W005']
+    tiles = parse_france_list('data/france_urls.txt')
     s3 = boto3.client('s3')
 
     catalog = create_stac_catalog()

From efaca69e3d8861d6deee387c97e22bbb84cac11c Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Thu, 27 Oct 2022 16:17:36 +0000
Subject: [PATCH 009/144] update looks

---
 src/coherence_stac.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index 73c03689..e37e3235 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -137,7 +137,6 @@ def create_stac_item(yearly_assets, seasonal_assets):
         properties=properties,
     )
 
-    # TODO don't know if look properties are correct
     ext_sar = sar.SarExtension.ext(item, add_if_missing=True)
     ext_sar.apply(
         'IW',
@@ -145,8 +144,8 @@ def create_stac_item(yearly_assets, seasonal_assets):
         [sar.Polarization('VV'), sar.Polarization('VH')],
         'COH',
         SENTINEL1_CENTER_FREQUENCY,
-        looks_range=7,
-        looks_azimuth=12,
+        looks_range=12,
+        looks_azimuth=3,
         observation_direction=sar.ObservationDirection('right'),
     )
 
@@ -231,7 +230,6 @@ def save_stac_catalog_s3(catalog, s3_client, bucket, key):
 
 
 def parse_france_list(data_path):
-    breakpoint()
     with open(data_path, 'r') as f:
         urls = [x.strip() for x in f.readlines()]
     tileids = [parse_url(x)['tileid'] for x in urls]

From 0211354b471c6c16570a6a035252ec10dc7b835b Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Thu, 27 Oct 2022 11:32:01 -0500
Subject: [PATCH 010/144] add multithreading

---
 src/coherence_stac.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index e37e3235..8f169b42 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -1,4 +1,6 @@
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
+from itertools import repeat
 from pathlib import Path
 
 import boto3
@@ -240,14 +242,26 @@ def parse_france_list(data_path):
     bucket = 'sentinel-1-global-coherence-earthbigdata'
     # tiles = ['N48W005', 'N49W005']
     tiles = parse_france_list('data/france_urls.txt')
+    prefixes = [f'data/tiles/{x}/' for x in tiles]
     s3 = boto3.client('s3')
 
+    # Multi-thread
+    print('creating...')
+    with ThreadPoolExecutor(max_workers=20) as executor:
+        results = list(
+            tqdm(executor.map(create_tile_stac_collection, repeat(s3), repeat(bucket), prefixes), total=len(prefixes))
+        )
     catalog = create_stac_catalog()
-    for tile in tiles:
-        prefix = f'data/tiles/{tile}/'
-        collection = create_tile_stac_collection(s3, bucket, prefix)
+    for collection in results:
         catalog.add_child(collection)
 
+    # # Single Thread
+    # catalog = create_stac_catalog()
+    # for tile in tiles:
+    #     prefix = f'data/tiles/{tile}/'
+    #     collection = create_tile_stac_collection(s3, bucket, prefix)
+    #     catalog.add_child(collection)
+
     upload_bucket = 'ffwilliams2-shenanigans'
     upload_key = 'stac/coherence_stac'
     save_stac_catalog_s3(catalog, s3, upload_bucket, upload_key)

From 756b8e49fff275ba5cd1a092fe35558d5fb7b7b3 Mon Sep 17 00:00:00 2001
From: Forrest Williams <ffwilliams2@alaska.edu>
Date: Thu, 27 Oct 2022 11:49:05 -0500
Subject: [PATCH 011/144] add multithreading for upload

---
 src/coherence_stac.py | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/coherence_stac.py b/src/coherence_stac.py
index 8f169b42..52d198e6 100644
--- a/src/coherence_stac.py
+++ b/src/coherence_stac.py
@@ -224,11 +224,7 @@ def save_stac_catalog_s3(catalog, s3_client, bucket, key):
     catalog_name = base_url.name
     catalog.normalize_hrefs(str(base_url))
     catalog.save(dest_href=catalog_name)
-    jsons = [x for x in Path(catalog_name).glob('**/*json')]
-    print('uploading...')
-    for json in tqdm(jsons):
-        s3_client.upload_file(str(json), bucket, str(Path(key).parent / json))
-    return f'{catalog_name}/catalog.json'
+    return catalog_name
 
 
 def parse_france_list(data_path):
@@ -240,6 +236,8 @@ def parse_france_list(data_path):
 
 if __name__ == '__main__':
     bucket = 'sentinel-1-global-coherence-earthbigdata'
+    upload_bucket = 'ffwilliams2-shenanigans'
+    upload_key = 'stac/coherence_stac'
     # tiles = ['N48W005', 'N49W005']
     tiles = parse_france_list('data/france_urls.txt')
     prefixes = [f'data/tiles/{x}/' for x in tiles]
@@ -255,13 +253,9 @@ def parse_france_list(data_path):
     for collection in results:
         catalog.add_child(collection)
 
-    # # Single Thread
-    # catalog = create_stac_catalog()
-    # for tile in tiles:
-    #     prefix = f'data/tiles/{tile}/'
-    #     collection = create_tile_stac_collection(s3, bucket, prefix)
-    #     catalog.add_child(collection)
-
-    upload_bucket = 'ffwilliams2-shenanigans'
-    upload_key = 'stac/coherence_stac'
-    save_stac_catalog_s3(catalog, s3, upload_bucket, upload_key)
+    catalog_name = save_stac_catalog_s3(catalog, s3, upload_bucket, upload_key)
+    jsons = [str(x) for x in Path(catalog_name).glob('**/*json')]
+    json_keys = [str(Path(upload_key).parent / x) for x in jsons]
+    print('uploading...')
+    with ThreadPoolExecutor(max_workers=20) as executor:
+        _ = list(tqdm(executor.map(s3.upload_file, jsons, repeat(upload_bucket), json_keys), total=len(jsons)))

From 1ee21991c6c5b2b52996d214034943d57a7816b3 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 31 Oct 2022 16:18:49 -0800
Subject: [PATCH 012/144] Add ingest_data script

---
 environment.yml |  1 +
 ingest_data.py  | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 ingest_data.py

diff --git a/environment.yml b/environment.yml
index b9c8862b..e3c1c128 100644
--- a/environment.yml
+++ b/environment.yml
@@ -9,6 +9,7 @@ dependencies:
   - python
   - pip
   - pystac
+  - requests
   - shapely
   - tqdm
   # For packaging, and testing
diff --git a/ingest_data.py b/ingest_data.py
new file mode 100644
index 00000000..d936de78
--- /dev/null
+++ b/ingest_data.py
@@ -0,0 +1,52 @@
+import argparse
+import json
+from pathlib import Path
+from urllib.parse import urljoin
+
+import requests
+
+
+def traverse(stac_object_path: Path, api_url: str) -> None:
+    with open(stac_object_path, 'r') as f:
+        stac_object: dict = json.load(f)
+
+    print(f'Adding STAC object {stac_object_path}')
+    add_stac_object(stac_object, api_url)
+
+    for child_path in get_child_paths(stac_object, stac_object_path.parent):
+        traverse(child_path, api_url)
+
+
+def get_child_paths(stac_object: dict, parent_path: Path) -> list[Path]:
+    # Assumes relative links.
+    return [
+        parent_path / link_object['href']
+        for link_object in stac_object['links'] if link_object['rel'] in ('child', 'item')
+    ]
+
+
+def add_stac_object(stac_object: dict, api_url: str) -> None:
+    if stac_object['type'] in ('Catalog', 'Collection'):  # TODO: is there a separate POST /catalogs endpoint?
+        endpoint = '/collections'
+    else:
+        assert stac_object['type'] == 'Feature'
+        endpoint = f'/collections/{stac_object["collection"]}/items'
+    url = urljoin(api_url, endpoint)
+    response = requests.post(url, json=stac_object)
+    response.raise_for_status()
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description='Add STAC catalog data to a STAC API application.')
+    parser.add_argument('catalog_path')
+    parser.add_argument('api_url')
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = parse_args()
+    traverse(Path(args.catalog_path), args.api_url)
+
+
+if __name__ == '__main__':
+    main()

From cc97562cd1d70fd23998b5b5981a08d26c4b9f06 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 1 Nov 2022 13:56:25 -0800
Subject: [PATCH 013/144] Update ingest script help text

---
 ingest_data.py | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/ingest_data.py b/ingest_data.py
index d936de78..fa064870 100644
--- a/ingest_data.py
+++ b/ingest_data.py
@@ -1,3 +1,16 @@
+"""
+Adds a STAC dataset to a STAC API application.
+
+Assumes that the dataset is arranged as a tree (connected acyclic
+graph). This means that all objects should be reachable from the root
+object and there should be no cycles.
+
+Assumes that links to child objects are specified as relative
+filesystem paths.
+
+Assumes that the STAC API supports the Transaction extension.
+"""
+
 import argparse
 import json
 from pathlib import Path
@@ -37,15 +50,15 @@ def add_stac_object(stac_object: dict, api_url: str) -> None:
 
 
 def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description='Add STAC catalog data to a STAC API application.')
-    parser.add_argument('catalog_path')
-    parser.add_argument('api_url')
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('root_object_path', help='Filesystem path to the root STAC object.')
+    parser.add_argument('api_url', help='URL for the STAC API.')
     return parser.parse_args()
 
 
 def main() -> None:
     args = parse_args()
-    traverse(Path(args.catalog_path), args.api_url)
+    traverse(Path(args.root_object_path), args.api_url)
 
 
 if __name__ == '__main__':

From a387ff6479d8189b757d69e1a324cbaef20a43c5 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 1 Nov 2022 14:15:06 -0800
Subject: [PATCH 014/144] Remove a comment

---
 ingest_data.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ingest_data.py b/ingest_data.py
index fa064870..a9282a6b 100644
--- a/ingest_data.py
+++ b/ingest_data.py
@@ -31,7 +31,6 @@ def traverse(stac_object_path: Path, api_url: str) -> None:
 
 
 def get_child_paths(stac_object: dict, parent_path: Path) -> list[Path]:
-    # Assumes relative links.
     return [
         parent_path / link_object['href']
         for link_object in stac_object['links'] if link_object['rel'] in ('child', 'item')

From 06c6f1f576b7cb6cf91403ba1e8d26585c4f1bf2 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 1 Nov 2022 14:40:22 -0800
Subject: [PATCH 015/144] Remove a TODO

---
 ingest_data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ingest_data.py b/ingest_data.py
index a9282a6b..eb1eaa1e 100644
--- a/ingest_data.py
+++ b/ingest_data.py
@@ -38,7 +38,7 @@ def get_child_paths(stac_object: dict, parent_path: Path) -> list[Path]:
 
 
 def add_stac_object(stac_object: dict, api_url: str) -> None:
-    if stac_object['type'] in ('Catalog', 'Collection'):  # TODO: is there a separate POST /catalogs endpoint?
+    if stac_object['type'] in ('Catalog', 'Collection'):
         endpoint = '/collections'
     else:
         assert stac_object['type'] == 'Feature'

From 0b36746e78fd701265fcfab3921f169a14de05c4 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 2 Nov 2022 12:13:27 -0800
Subject: [PATCH 016/144] Start adding cloudformation for api frontend

---
 cloudformation.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 cloudformation.yml

diff --git a/cloudformation.yml b/cloudformation.yml
new file mode 100644
index 00000000..d8fd1e84
--- /dev/null
+++ b/cloudformation.yml
@@ -0,0 +1,10 @@
+Resources:
+  Cluster:
+    Type: AWS::ECS::Cluster
+
+  TaskDefinition:
+    Type: AWS::ECS::TaskDefinition
+    Properties:
+      ContainerDefinitions:
+        - Name: stac-fastapi
+          Image: '' # TODO
\ No newline at end of file

From 0d3d8b630bdcbc1818522c541e480499c4d513fb Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 3 Nov 2022 09:52:22 -0800
Subject: [PATCH 017/144] Move api cf template

---
 cloudformation.yml => apps/api/cloudformation.yml | 2 ++
 1 file changed, 2 insertions(+)
 rename cloudformation.yml => apps/api/cloudformation.yml (84%)

diff --git a/cloudformation.yml b/apps/api/cloudformation.yml
similarity index 84%
rename from cloudformation.yml
rename to apps/api/cloudformation.yml
index d8fd1e84..0ffd4354 100644
--- a/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -1,3 +1,5 @@
+AWSTemplateFormatVersion: 2010-09-09
+
 Resources:
   Cluster:
     Type: AWS::ECS::Cluster

From e931bd452b9faf99940e20d62fe3ce7b948b128a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 3 Nov 2022 16:17:02 -0800
Subject: [PATCH 018/144] Start adding backend database

---
 .gitignore                       |  2 ++
 Makefile                         | 23 +++++++++++++++++++++++
 README.md                        |  9 +++++++++
 apps/database/cloudformation.yml |  3 +++
 environment.yml                  |  1 +
 5 files changed, 38 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 Makefile
 create mode 100644 apps/database/cloudformation.yml

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..2c98991e
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+# CloudFormation
+packaged.yml
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..89d41001
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,23 @@
+# TODO create target for installing dependencies
+
+template_path ?= '' # TODO
+s3_bucket ?= '' # TODO
+stack_name ?= '' # TODO
+# TODO allow adding --role-arn option to deploy command, for cicd
+deploy:
+	aws cloudformation package \
+	    --template-file ${template_path} \
+	    --s3-bucket ${s3_bucket} \
+	    --output-template-file packaged.yml \
+	&& aws cloudformation deploy \
+	    --template-file packaged.yml \
+	    --stack-name ${stack_name} \
+	    --capabilities CAPABILITY_NAMED_IAM
+
+static: flake8 cfn-lint
+
+flake8:
+	flake8 --max-line-length=120
+
+cfn-lint:
+	cfn-lint --template `find . -name cloudformation.yml` --info
diff --git a/README.md b/README.md
index 1c486c21..ac6e4ce6 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,12 @@
 # asf-stac
 
 A repository containing code related to the creation and hosting of STAC catalogs by the ASF tools team.
+
+TODO: document connecting to the database with `psql`
+
+TODO: move these PostGIS installation commands into a script somewhere
+([source](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.PostGIS.html):
+
+```
+CREATE EXTENSION postgis;
+```
\ No newline at end of file
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
new file mode 100644
index 00000000..79715f4e
--- /dev/null
+++ b/apps/database/cloudformation.yml
@@ -0,0 +1,3 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Resources:
\ No newline at end of file
diff --git a/environment.yml b/environment.yml
index e3c1c128..7ec78ac9 100644
--- a/environment.yml
+++ b/environment.yml
@@ -6,6 +6,7 @@ channels:
 dependencies:
   - boto3
   - botocore
+  - cfn-lint
   - python
   - pip
   - pystac

From f709965a56bcedb21fdd340264fc74eee5209641 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 4 Nov 2022 12:04:57 -0800
Subject: [PATCH 019/144] Add database cloudformation

---
 Makefile                         | 16 +++++++++--
 README.md                        | 48 +++++++++++++++++++++++++++++---
 apps/database/cloudformation.yml | 34 +++++++++++++++++++++-
 requirements.txt                 |  1 +
 4 files changed, 92 insertions(+), 7 deletions(-)
 create mode 100644 requirements.txt

diff --git a/Makefile b/Makefile
index 89d41001..95b6a68e 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,5 @@
-# TODO create target for installing dependencies
+install:
+	python -m pip install -r requirements.txt
 
 template_path ?= '' # TODO
 s3_bucket ?= '' # TODO
@@ -12,7 +13,18 @@ deploy:
 	&& aws cloudformation deploy \
 	    --template-file packaged.yml \
 	    --stack-name ${stack_name} \
-	    --capabilities CAPABILITY_NAMED_IAM
+	    --capabilities CAPABILITY_NAMED_IAM \
+	    --parameter-overrides \
+	      AsfNetworkCidr=${ASF_NETWORK_CIDR} \
+	      DatabasePassword=${DATABASE_PASSWORD}
+
+db_host ?= 'host'
+db_password ?= 'password'
+psql:
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql
+
+migrate:
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} pypgstac migrate
 
 static: flake8 cfn-lint
 
diff --git a/README.md b/README.md
index ac6e4ce6..64ab4993 100644
--- a/README.md
+++ b/README.md
@@ -2,11 +2,51 @@
 
 A repository containing code related to the creation and hosting of STAC catalogs by the ASF tools team.
 
-TODO: document connecting to the database with `psql`
+## Developer setup
 
-TODO: move these PostGIS installation commands into a script somewhere
-([source](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.PostGIS.html):
+TODO: document installing developer deps
+
+## STAC API
+
+### Manual deployment
+
+TODO: document deploying with `make`
+
+### Connecting to the database
+
+Confirm you have the `psql` command installed, then run:
+
+```
+make psql db_host=<host> db_password=<password>
+```
+
+You can find the appropriate value for `<host>` by navigating to the database instance via the CloudFormation or
+RDS console and copying the "Endpoint" field.
+
+### Installing the PostGIS extension
+
+TODO: automate this step
+
+After deploying the database for the first time, connect to the database with `psql` as described above, then run:
 
 ```
 CREATE EXTENSION postgis;
-```
\ No newline at end of file
+```
+
+### Installing PGStac
+
+TODO: automate this step (installation and upgrades)
+
+Next, install PGStac into the database:
+
+```
+make migrate db_host=<host> db_password=<password>
+```
+
+After upgrading `pypgstac`, you can re-run the above command to upgrade the version of PGStac installed to the database.
+
+For more information about migrations, see <https://stac-utils.github.io/pgstac/pypgstac/>.
+
+### Ingesting STAC dataset
+
+TODO: document using ingest script
\ No newline at end of file
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 79715f4e..6936c97b 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -1,3 +1,35 @@
 AWSTemplateFormatVersion: 2010-09-09
 
-Resources:
\ No newline at end of file
+Parameters:
+  AsfNetworkCidr:
+    Type: String
+    NoEcho: true
+
+  DatabasePassword:
+    Type: String
+    NoEcho: true
+
+Resources:
+  DatabaseInstance:
+    Type: AWS::RDS::DBInstance
+    Properties:
+      AllocatedStorage: '20' # TODO depends on StorageType?
+      DBInstanceClass: 'db.t3.micro' # TODO different class for prod?
+      DBSecurityGroups:
+        - !Ref DatabaseSecurityGroup
+      Engine: postgres
+      EngineVersion: '14.4'
+      MasterUsername: postgres
+      MasterUserPassword: !Ref DatabasePassword
+      # TODO: MaxAllocatedStorage: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-maxallocatedstorage
+      Port: '5432'
+      # TODO: ProcessorFeatures: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-processorfeatures
+      PubliclyAccessible: true
+      # TODO: StorageType: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-storagetype
+
+  DatabaseSecurityGroup:
+    Type: AWS::RDS::DBSecurityGroup
+    Properties:
+      DBSecurityGroupIngress:
+        - CIDRIP: !Ref AsfNetworkCidr
+      GroupDescription: Allows incoming connections from within the ASF network.
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..e78c36b4
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1 @@
+pypgstac[psycopg]

From 5f9989aabe54a19784dd204854f2e04fe9cd1412 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 4 Nov 2022 15:06:15 -0800
Subject: [PATCH 020/144] Update README and requirements

---
 README.md        | 28 ++--------------------------
 environment.yml  | 23 -----------------------
 requirements.txt |  8 ++++++++
 3 files changed, 10 insertions(+), 49 deletions(-)
 delete mode 100644 environment.yml

diff --git a/README.md b/README.md
index 64ab4993..f0961a8e 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ A repository containing code related to the creation and hosting of STAC catalog
 
 ## Developer setup
 
-TODO: document installing developer deps
+TODO: document creating conda env and installing developer deps
 
 ## STAC API
 
@@ -23,30 +23,6 @@ make psql db_host=<host> db_password=<password>
 You can find the appropriate value for `<host>` by navigating to the database instance via the CloudFormation or
 RDS console and copying the "Endpoint" field.
 
-### Installing the PostGIS extension
-
-TODO: automate this step
-
-After deploying the database for the first time, connect to the database with `psql` as described above, then run:
-
-```
-CREATE EXTENSION postgis;
-```
-
-### Installing PGStac
-
-TODO: automate this step (installation and upgrades)
-
-Next, install PGStac into the database:
-
-```
-make migrate db_host=<host> db_password=<password>
-```
-
-After upgrading `pypgstac`, you can re-run the above command to upgrade the version of PGStac installed to the database.
-
-For more information about migrations, see <https://stac-utils.github.io/pgstac/pypgstac/>.
-
 ### Ingesting STAC dataset
 
-TODO: document using ingest script
\ No newline at end of file
+TODO: document using ingest script
diff --git a/environment.yml b/environment.yml
deleted file mode 100644
index 7ec78ac9..00000000
--- a/environment.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: asf-stac
-channels:
-  - hyp3
-  - conda-forge
-  - nodefaults
-dependencies:
-  - boto3
-  - botocore
-  - cfn-lint
-  - python
-  - pip
-  - pystac
-  - requests
-  - shapely
-  - tqdm
-  # For packaging, and testing
-  - flake8
-  - flake8-import-order
-  - flake8-blind-except
-  - flake8-builtins
-  - pytest
-  - pytest-console-scripts
-  - pytest-cov
diff --git a/requirements.txt b/requirements.txt
index e78c36b4..413d4264 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,9 @@
+# TODO pin versions
+boto3
+cfn-lint
 pypgstac[psycopg]
+pystac
+requests
+shapely
+tqdm
+flake8

From cba41b75bc08e52cc8d03505fc6452cfe8236679 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 4 Nov 2022 15:25:53 -0800
Subject: [PATCH 021/144] alphabetize deps

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 413d4264..3bd1c021 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,9 @@
 # TODO pin versions
 boto3
 cfn-lint
+flake8
 pypgstac[psycopg]
 pystac
 requests
 shapely
 tqdm
-flake8

From 3ea5072c85bdf288c115b19b22166b28bba29d3f Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 4 Nov 2022 16:25:00 -0800
Subject: [PATCH 022/144] Allow running stac api locally

---
 Makefile         |  3 +++
 README.md        | 13 ++++++++++++-
 requirements.txt |  9 +++++++++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 95b6a68e..d91377a0 100644
--- a/Makefile
+++ b/Makefile
@@ -26,6 +26,9 @@ psql:
 migrate:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} pypgstac migrate
 
+run-api:
+	POSTGRES_HOST_READER=${db_host} POSTGRES_HOST_WRITER=${db_host} POSTGRES_PORT=5432 POSTGRES_DBNAME=postgres POSTGRES_USER=postgres POSTGRES_PASS=${db_password} python -m stac_fastapi.pgstac.app
+
 static: flake8 cfn-lint
 
 flake8:
diff --git a/README.md b/README.md
index f0961a8e..da0a4587 100644
--- a/README.md
+++ b/README.md
@@ -23,6 +23,17 @@ make psql db_host=<host> db_password=<password>
 You can find the appropriate value for `<host>` by navigating to the database instance via the CloudFormation or
 RDS console and copying the "Endpoint" field.
 
+### Running the API locally
+
+To run the STAC API locally:
+
+```
+make run-api db_host=<host> db_password=<password>
+```
+
+You should see something like `Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)` in the output; you can
+query the API at that URL.
+
 ### Ingesting STAC dataset
 
-TODO: document using ingest script
+Run `python ingest_data.py -h` for usage instructions.
diff --git a/requirements.txt b/requirements.txt
index 3bd1c021..648b8d9d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 # TODO pin versions
+
 boto3
 cfn-lint
 flake8
@@ -7,3 +8,11 @@ pystac
 requests
 shapely
 tqdm
+
+# TODO move to separate requirements file for Docker image:
+pygeofilter==0.2.0
+stac-fastapi.api
+stac-fastapi.types
+stac-fastapi.extensions
+stac-fastapi.pgstac
+uvicorn

From 9d18ddd8924671c43ea58696e4230b8233d30b1f Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 7 Nov 2022 11:58:51 -0900
Subject: [PATCH 023/144] Deploy full api stack

---
 .gitignore                       |   4 ++
 Makefile                         |  18 +++---
 apps/api/cloudformation.yml      | 103 ++++++++++++++++++++++++++++---
 apps/api/requirements.txt        |   6 ++
 apps/api/src/api.py              |   4 ++
 apps/cloudformation.yml          |  27 ++++++++
 apps/database/cloudformation.yml |   5 ++
 requirements.txt                 |  31 ++++------
 8 files changed, 166 insertions(+), 32 deletions(-)
 create mode 100644 apps/api/requirements.txt
 create mode 100644 apps/api/src/api.py
 create mode 100644 apps/cloudformation.yml

diff --git a/.gitignore b/.gitignore
index 2c98991e..13a7c943 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,6 @@
 # CloudFormation
 packaged.yml
+
+# Lambda dependencies
+apps/api/src/*
+!apps/api/src/api.py
diff --git a/Makefile b/Makefile
index d91377a0..05678a6d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,13 +1,17 @@
 install:
+	python -m pip install --upgrade pip && \
 	python -m pip install -r requirements.txt
 
-template_path ?= '' # TODO
-s3_bucket ?= '' # TODO
-stack_name ?= '' # TODO
+install-lambda-deps:
+	python -m pip install --upgrade pip && \
+	python -m pip install -r apps/api/requirements.txt -t apps/api/src/
+
+s3_bucket ?= ''
+stack_name ?= ''
 # TODO allow adding --role-arn option to deploy command, for cicd
 deploy:
 	aws cloudformation package \
-	    --template-file ${template_path} \
+	    --template-file apps/cloudformation.yml \
 	    --s3-bucket ${s3_bucket} \
 	    --output-template-file packaged.yml \
 	&& aws cloudformation deploy \
@@ -18,8 +22,8 @@ deploy:
 	      AsfNetworkCidr=${ASF_NETWORK_CIDR} \
 	      DatabasePassword=${DATABASE_PASSWORD}
 
-db_host ?= 'host'
-db_password ?= 'password'
+db_host ?= ''
+db_password ?= ''
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql
 
@@ -35,4 +39,4 @@ flake8:
 	flake8 --max-line-length=120
 
 cfn-lint:
-	cfn-lint --template `find . -name cloudformation.yml` --info
+	cfn-lint --template `find . -name cloudformation.yml` --info --ignore-checks W3002
diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 0ffd4354..429e856b 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -1,12 +1,101 @@
 AWSTemplateFormatVersion: 2010-09-09
 
+Parameters:
+  DatabaseHost:
+    Type: String
+    NoEcho: true
+
+  DatabasePassword:
+    Type: String
+    NoEcho: true
+
+# TODO the lambda should run inside the default VPC
 Resources:
-  Cluster:
-    Type: AWS::ECS::Cluster
+  Lambda:
+    Type: AWS::Lambda::Function
+    Properties:
+      Environment:
+        Variables:
+          POSTGRES_HOST_READER: !Ref DatabaseHost
+          POSTGRES_HOST_WRITER: !Ref DatabaseHost
+          POSTGRES_PORT: 5432
+          POSTGRES_DBNAME: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_PASS: !Ref DatabasePassword
+      Code: src/
+      Handler: api.handler
+      Role: !GetAtt LambdaRole.Arn
+      Runtime: python3.9
+      Timeout: 30
+
+  LambdaLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/lambda/${Lambda}"
+      RetentionInDays: 90
+
+  LambdaRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: lambda.amazonaws.com
+          Effect: Allow
+      ManagedPolicyArns:
+        - !Ref LambdaPolicy
+
+  LambdaPolicy:
+    Type: AWS::IAM::ManagedPolicy
+    Properties:
+      PolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          - Effect: Allow
+            Action:
+              - logs:CreateLogStream
+              - logs:PutLogEvents
+            Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*"
+
+  Api:
+    Type: AWS::ApiGatewayV2::Api
+    Properties:
+      Name: !Ref AWS::StackName
+      ProtocolType: HTTP
+      Target: !GetAtt Lambda.Arn
+      CredentialsArn: !GetAtt ApiRole.Arn
+
+  ApiOverrides:
+    Type: AWS::ApiGatewayV2::ApiGatewayManagedOverrides
+    Properties:
+      ApiId: !Ref Api
+      Stage:
+        AccessLogSettings:
+          DestinationArn: !GetAtt ApiLogGroup.Arn
+          Format: '{"sourceIp":"$context.identity.sourceIp","httpMethod":"$context.httpMethod","path":"$context.path","status":"$context.status","responseLength":"$context.responseLength","responseLatency":"$context.responseLatency","requestTime":"$context.requestTime","protocol":"$context.protocol","userAgent":"$context.identity.userAgent","requestId":"$context.requestId"}'
+
+  ApiLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      RetentionInDays: 180
 
-  TaskDefinition:
-    Type: AWS::ECS::TaskDefinition
+  ApiRole:
+    Type: AWS::IAM::Role
     Properties:
-      ContainerDefinitions:
-        - Name: stac-fastapi
-          Image: '' # TODO
\ No newline at end of file
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: apigateway.amazonaws.com
+          Effect: Allow
+      Policies:
+        - PolicyName: policy
+          PolicyDocument:
+            Version: 2012-10-17
+            Statement:
+              - Effect: Allow
+                Action: lambda:InvokeFunction
+                Resource: !GetAtt Lambda.Arn
\ No newline at end of file
diff --git a/apps/api/requirements.txt b/apps/api/requirements.txt
new file mode 100644
index 00000000..799a607a
--- /dev/null
+++ b/apps/api/requirements.txt
@@ -0,0 +1,6 @@
+mangum==0.16.0
+pygeofilter==0.2.0
+stac-fastapi.api==2.4.1
+stac-fastapi.extensions==2.4.1
+stac-fastapi.pgstac==2.3.0
+stac-fastapi.types==2.4.1
diff --git a/apps/api/src/api.py b/apps/api/src/api.py
new file mode 100644
index 00000000..9cc0dd45
--- /dev/null
+++ b/apps/api/src/api.py
@@ -0,0 +1,4 @@
+from mangum import Mangum
+from stac_fastapi.pgstac.app import app
+
+handler = Mangum(app)
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
new file mode 100644
index 00000000..54352644
--- /dev/null
+++ b/apps/cloudformation.yml
@@ -0,0 +1,27 @@
+AWSTemplateFormatVersion: 2010-09-09
+
+Parameters:
+  AsfNetworkCidr:
+    Type: String
+    NoEcho: true
+
+  DatabasePassword:
+    Type: String
+    NoEcho: true
+
+Resources:
+  Api:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        DatabaseHost: !GetAtt Database.Outputs.DatabaseHost
+        DatabasePassword: !Ref DatabasePassword
+      TemplateURL: api/cloudformation.yml
+
+  Database:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      Parameters:
+        AsfNetworkCidr: !Ref AsfNetworkCidr
+        DatabasePassword: !Ref DatabasePassword
+      TemplateURL: database/cloudformation.yml
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 6936c97b..2c5361e9 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -9,6 +9,11 @@ Parameters:
     Type: String
     NoEcho: true
 
+Outputs:
+  DatabaseHost:
+    Value: !GetAtt DatabaseInstance.Endpoint.Address
+
+# TODO the database should also accept connections from w/in the default VPC
 Resources:
   DatabaseInstance:
     Type: AWS::RDS::DBInstance
diff --git a/requirements.txt b/requirements.txt
index 648b8d9d..e901fea9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,18 +1,13 @@
-# TODO pin versions
-
-boto3
-cfn-lint
-flake8
-pypgstac[psycopg]
-pystac
-requests
-shapely
-tqdm
-
-# TODO move to separate requirements file for Docker image:
-pygeofilter==0.2.0
-stac-fastapi.api
-stac-fastapi.types
-stac-fastapi.extensions
-stac-fastapi.pgstac
-uvicorn
+-r apps/api/requirements.txt
+boto3==1.26.3
+cfn-lint==0.71.0
+flake8==5.0.4
+psycopg==3.0.18
+psycopg-binary==3.0.18
+psycopg-pool==3.1.4
+pypgstac==0.6.10
+pystac==1.6.1
+requests==2.28.1
+shapely==1.8.5.post1
+tqdm==4.64.1
+uvicorn==0.19.0

From 783eff6cee46558239ef223b97c7836f3ddb5949 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 7 Nov 2022 16:34:05 -0900
Subject: [PATCH 024/144] Open database to all connections

---
 Makefile                         |  1 -
 apps/api/cloudformation.yml      |  1 -
 apps/cloudformation.yml          |  5 -----
 apps/database/cloudformation.yml | 10 +++-------
 4 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/Makefile b/Makefile
index 05678a6d..3f0c2520 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,6 @@ deploy:
 	    --stack-name ${stack_name} \
 	    --capabilities CAPABILITY_NAMED_IAM \
 	    --parameter-overrides \
-	      AsfNetworkCidr=${ASF_NETWORK_CIDR} \
 	      DatabasePassword=${DATABASE_PASSWORD}
 
 db_host ?= ''
diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 429e856b..4878be22 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -9,7 +9,6 @@ Parameters:
     Type: String
     NoEcho: true
 
-# TODO the lambda should run inside the default VPC
 Resources:
   Lambda:
     Type: AWS::Lambda::Function
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 54352644..783492ed 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -1,10 +1,6 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
-  AsfNetworkCidr:
-    Type: String
-    NoEcho: true
-
   DatabasePassword:
     Type: String
     NoEcho: true
@@ -22,6 +18,5 @@ Resources:
     Type: AWS::CloudFormation::Stack
     Properties:
       Parameters:
-        AsfNetworkCidr: !Ref AsfNetworkCidr
         DatabasePassword: !Ref DatabasePassword
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 2c5361e9..abf532a1 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -1,10 +1,6 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
-  AsfNetworkCidr:
-    Type: String
-    NoEcho: true
-
   DatabasePassword:
     Type: String
     NoEcho: true
@@ -13,7 +9,6 @@ Outputs:
   DatabaseHost:
     Value: !GetAtt DatabaseInstance.Endpoint.Address
 
-# TODO the database should also accept connections from w/in the default VPC
 Resources:
   DatabaseInstance:
     Type: AWS::RDS::DBInstance
@@ -32,9 +27,10 @@ Resources:
       PubliclyAccessible: true
       # TODO: StorageType: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-storagetype
 
+  # TODO only allow incoming connections from within the ASF network and from our Lambda function
   DatabaseSecurityGroup:
     Type: AWS::RDS::DBSecurityGroup
     Properties:
       DBSecurityGroupIngress:
-        - CIDRIP: !Ref AsfNetworkCidr
-      GroupDescription: Allows incoming connections from within the ASF network.
+        - CIDRIP: 0.0.0.0/0
+      GroupDescription: STAC API database security group.

From 3368501acbd0860c13b72040f2bc24f66e0fffff Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 7 Nov 2022 16:36:53 -0900
Subject: [PATCH 025/144] Simplify api handler

---
 apps/api/src/api.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/apps/api/src/api.py b/apps/api/src/api.py
index 9cc0dd45..a537b4b0 100644
--- a/apps/api/src/api.py
+++ b/apps/api/src/api.py
@@ -1,4 +1 @@
-from mangum import Mangum
-from stac_fastapi.pgstac.app import app
-
-handler = Mangum(app)
+from stac_fastapi.pgstac.app import handler  # noqa: F401

From b71e059a23b0987eb0a9b75fbafc354937764332 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 15:18:19 -0900
Subject: [PATCH 026/144] Add deployment workflow

---
 .github/workflows/deploy-stac-api.yml | 38 +++++++++++++++++++++++++++
 Makefile                              |  5 +++-
 README.md                             |  4 ---
 3 files changed, 42 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/deploy-stac-api.yml

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
new file mode 100644
index 00000000..43bbd42c
--- /dev/null
+++ b/.github/workflows/deploy-stac-api.yml
@@ -0,0 +1,38 @@
+name: Deploy STAC API
+
+on:
+  push:
+    branches:
+      - sandbox
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CF_TEMPLATES_BUCKET: ${{ secrets.CF_TEMPLATES_BUCKET }}
+  CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
+  DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
+  STACK_NAME: stac-api-sandbox
+
+jobs:
+  deploy-stac-api:
+    runs-on: ubuntu-latest
+    environment: sandbox
+    steps:
+      - uses: actions/checkout@v3
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+      - name: Install Lambda dependencies
+        run: make install-lambda-deps
+      - name: Deploy CloudFormation stack
+        run: |
+          make deploy \
+            s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
+            stack_name=${{ env.STACK_NAME }} \
+            cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
+            new_db_password=${{ env.DATABASE_PASSWORD }}
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 3f0c2520..97999385 100644
--- a/Makefile
+++ b/Makefile
@@ -8,6 +8,8 @@ install-lambda-deps:
 
 s3_bucket ?= ''
 stack_name ?= ''
+cloudformation_role_arn ?= ''
+new_db_password ?= ''
 # TODO allow adding --role-arn option to deploy command, for cicd
 deploy:
 	aws cloudformation package \
@@ -18,8 +20,9 @@ deploy:
 	    --template-file packaged.yml \
 	    --stack-name ${stack_name} \
 	    --capabilities CAPABILITY_NAMED_IAM \
+	    --role-arn ${cloudformation_role_arn} \
 	    --parameter-overrides \
-	      DatabasePassword=${DATABASE_PASSWORD}
+	      DatabasePassword=${new_db_password}
 
 db_host ?= ''
 db_password ?= ''
diff --git a/README.md b/README.md
index da0a4587..d600f4aa 100644
--- a/README.md
+++ b/README.md
@@ -8,10 +8,6 @@ TODO: document creating conda env and installing developer deps
 
 ## STAC API
 
-### Manual deployment
-
-TODO: document deploying with `make`
-
 ### Connecting to the database
 
 Confirm you have the `psql` command installed, then run:

From 0cab1debbd25d4073cdabf86b29e5feb72ad0be0 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 16:09:00 -0900
Subject: [PATCH 027/144] Output database host from main cf stack

---
 apps/cloudformation.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 783492ed..69a43380 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -5,6 +5,10 @@ Parameters:
     Type: String
     NoEcho: true
 
+Outputs:
+  DatabaseHost:
+    Value: !GetAtt Database.Outputs.DatabaseHost
+
 Resources:
   Api:
     Type: AWS::CloudFormation::Stack

From 0e4a050240086f8465511bc5718e0ba80ce6bfe4 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 16:21:47 -0900
Subject: [PATCH 028/144] Install postgis

---
 .github/workflows/deploy-stac-api.yml | 11 ++++++++++-
 Makefile                              |  4 ++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 43bbd42c..752ddc1f 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -35,4 +35,13 @@ jobs:
             s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
             stack_name=${{ env.STACK_NAME }} \
             cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
-            new_db_password=${{ env.DATABASE_PASSWORD }}
\ No newline at end of file
+            new_db_password=${{ env.DATABASE_PASSWORD }}
+      - name: Install PostGIS
+        run: |
+          make install-postgis \
+            db_host=$(aws cloudformation describe-stacks \
+                      --stack-name $STACK_NAME \
+                      --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
+                      --output text) \
+            db_password=${{ env.DATABASE_PASSWORD }}
+      # TODO migrate command
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 97999385..50e2b656 100644
--- a/Makefile
+++ b/Makefile
@@ -29,6 +29,10 @@ db_password ?= ''
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql
 
+install-postgis:
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql \
+	       -c 'CREATE EXTENSION postgis;'
+
 migrate:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} pypgstac migrate
 

From 8f64002d86d731181d0449338c77857653ea78fd Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 16:28:01 -0900
Subject: [PATCH 029/144] add TODO

---
 .github/workflows/deploy-stac-api.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 752ddc1f..e81af6c3 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -30,6 +30,7 @@ jobs:
       - name: Install Lambda dependencies
         run: make install-lambda-deps
       - name: Deploy CloudFormation stack
+        # TODO prevent passwords from being echoed
         run: |
           make deploy \
             s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
@@ -44,4 +45,4 @@ jobs:
                       --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
                       --output text) \
             db_password=${{ env.DATABASE_PASSWORD }}
-      # TODO migrate command
\ No newline at end of file
+      # TODO migrate command

From a41738e05cc74e4c02337de9fdb4bb7bb4046d88 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 16:49:52 -0900
Subject: [PATCH 030/144] Migrate database

---
 .github/workflows/deploy-stac-api.yml | 19 +++++++++++--------
 Makefile                              |  9 ++++++---
 install-or-upgrade-postgis.sql        |  3 +++
 requirements-database-migrations.txt  |  4 ++++
 requirements.txt                      |  5 +----
 5 files changed, 25 insertions(+), 15 deletions(-)
 create mode 100644 install-or-upgrade-postgis.sql
 create mode 100644 requirements-database-migrations.txt

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index e81af6c3..34cbdc70 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -29,6 +29,8 @@ jobs:
           python-version: 3.9
       - name: Install Lambda dependencies
         run: make install-lambda-deps
+      - name: Install database migration dependencies
+        run: make install-db-migration-deps
       - name: Deploy CloudFormation stack
         # TODO prevent passwords from being echoed
         run: |
@@ -37,12 +39,13 @@ jobs:
             stack_name=${{ env.STACK_NAME }} \
             cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
             new_db_password=${{ env.DATABASE_PASSWORD }}
-      - name: Install PostGIS
+      - name: Get database host
         run: |
-          make install-postgis \
-            db_host=$(aws cloudformation describe-stacks \
-                      --stack-name $STACK_NAME \
-                      --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
-                      --output text) \
-            db_password=${{ env.DATABASE_PASSWORD }}
-      # TODO migrate command
+          DATABASE_HOST=$(aws cloudformation describe-stacks \
+                          --stack-name $STACK_NAME \
+                          --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
+                          --output text)
+      - name: Install or upgrade PostGIS
+        run: make install-or-upgrade-postgis db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
+      - name: Migrate database
+        run: make migrate db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
diff --git a/Makefile b/Makefile
index 50e2b656..7d2c38aa 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,10 @@ install-lambda-deps:
 	python -m pip install --upgrade pip && \
 	python -m pip install -r apps/api/requirements.txt -t apps/api/src/
 
+install-db-migration-deps:
+	python -m pip install --upgrade pip && \
+	python -m pip install -r requirements-database-migrations.txt
+
 s3_bucket ?= ''
 stack_name ?= ''
 cloudformation_role_arn ?= ''
@@ -29,9 +33,8 @@ db_password ?= ''
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql
 
-install-postgis:
-	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql \
-	       -c 'CREATE EXTENSION postgis;'
+install-or-upgrade-postgis:
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql -f install-or-upgrade-postgis.sql
 
 migrate:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} pypgstac migrate
diff --git a/install-or-upgrade-postgis.sql b/install-or-upgrade-postgis.sql
new file mode 100644
index 00000000..0c233820
--- /dev/null
+++ b/install-or-upgrade-postgis.sql
@@ -0,0 +1,3 @@
+-- TODO create extension with appropriate version: https://www.postgresql.org/docs/current/sql-createextension.html
+-- TODO upgrade extension to appropriate version
+CREATE EXTENSION IF NOT EXISTS postgis;
diff --git a/requirements-database-migrations.txt b/requirements-database-migrations.txt
new file mode 100644
index 00000000..7165dc9c
--- /dev/null
+++ b/requirements-database-migrations.txt
@@ -0,0 +1,4 @@
+psycopg==3.0.18
+psycopg-binary==3.0.18
+psycopg-pool==3.1.4
+pypgstac==0.6.10
diff --git a/requirements.txt b/requirements.txt
index e901fea9..6a997d5c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,11 +1,8 @@
+-r requirements-database-migrations.txt
 -r apps/api/requirements.txt
 boto3==1.26.3
 cfn-lint==0.71.0
 flake8==5.0.4
-psycopg==3.0.18
-psycopg-binary==3.0.18
-psycopg-pool==3.1.4
-pypgstac==0.6.10
 pystac==1.6.1
 requests==2.28.1
 shapely==1.8.5.post1

From 16c0c2a82011b7659b93ae17b73306e40acc9d34 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 16:58:33 -0900
Subject: [PATCH 031/144] set github env var the right way

---
 .github/workflows/deploy-stac-api.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 34cbdc70..cd7d3207 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -41,10 +41,10 @@ jobs:
             new_db_password=${{ env.DATABASE_PASSWORD }}
       - name: Get database host
         run: |
-          DATABASE_HOST=$(aws cloudformation describe-stacks \
-                          --stack-name $STACK_NAME \
-                          --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
-                          --output text)
+          echo "DATABASE_HOST=$(aws cloudformation describe-stacks \
+                                --stack-name $STACK_NAME \
+                                --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
+                                --output text)" >> $GITHUB_ENV
       - name: Install or upgrade PostGIS
         run: make install-or-upgrade-postgis db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
       - name: Migrate database

From f6012e6b5e2f4fbfc66f7f1622b6cd79f5b6dfb6 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Tue, 8 Nov 2022 17:05:26 -0900
Subject: [PATCH 032/144] disable make output for cicd

---
 .github/workflows/deploy-stac-api.yml | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index cd7d3207..4ca4ec4d 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -28,13 +28,12 @@ jobs:
         with:
           python-version: 3.9
       - name: Install Lambda dependencies
-        run: make install-lambda-deps
+        run: make -s install-lambda-deps
       - name: Install database migration dependencies
-        run: make install-db-migration-deps
+        run: make -s install-db-migration-deps
       - name: Deploy CloudFormation stack
-        # TODO prevent passwords from being echoed
         run: |
-          make deploy \
+          make -s deploy \
             s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
             stack_name=${{ env.STACK_NAME }} \
             cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
@@ -46,6 +45,6 @@ jobs:
                                 --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
                                 --output text)" >> $GITHUB_ENV
       - name: Install or upgrade PostGIS
-        run: make install-or-upgrade-postgis db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
+        run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
       - name: Migrate database
-        run: make migrate db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
+        run: make -s migrate db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}

From aae1ccd0393a39640eee747c7b2806737d32ba10 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 09:32:53 -0900
Subject: [PATCH 033/144] Finish postgis install/upgrade script

---
 install-or-upgrade-postgis.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/install-or-upgrade-postgis.sql b/install-or-upgrade-postgis.sql
index 0c233820..1a62f03a 100644
--- a/install-or-upgrade-postgis.sql
+++ b/install-or-upgrade-postgis.sql
@@ -1,3 +1,3 @@
--- TODO create extension with appropriate version: https://www.postgresql.org/docs/current/sql-createextension.html
--- TODO upgrade extension to appropriate version
-CREATE EXTENSION IF NOT EXISTS postgis;
+postgis_version CONSTANT string := '3.1.5';
+CREATE EXTENSION IF NOT EXISTS postgis WITH VERSION postgis_version;
+ALTER EXTENSION postgis UPDATE TO postgis_version;

From f8499e633a8f4dfabea01d9444ff42300f0a3668 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 09:56:35 -0900
Subject: [PATCH 034/144] fix postgis install script

---
 install-or-upgrade-postgis.sql | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/install-or-upgrade-postgis.sql b/install-or-upgrade-postgis.sql
index 1a62f03a..f701aa06 100644
--- a/install-or-upgrade-postgis.sql
+++ b/install-or-upgrade-postgis.sql
@@ -1,3 +1,5 @@
-postgis_version CONSTANT string := '3.1.5';
-CREATE EXTENSION IF NOT EXISTS postgis WITH VERSION postgis_version;
-ALTER EXTENSION postgis UPDATE TO postgis_version;
+\set ON_ERROR_STOP TRUE
+\set postgis_version '3.1.5'
+
+CREATE EXTENSION IF NOT EXISTS postgis WITH VERSION :'postgis_version';
+ALTER EXTENSION postgis UPDATE TO :'postgis_version';

From a09131a017c2d901f51c903610837176587782d3 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:13:36 -0900
Subject: [PATCH 035/144] change postgis version to test script failure

---
 install-or-upgrade-postgis.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install-or-upgrade-postgis.sql b/install-or-upgrade-postgis.sql
index f701aa06..55f464f8 100644
--- a/install-or-upgrade-postgis.sql
+++ b/install-or-upgrade-postgis.sql
@@ -1,5 +1,5 @@
 \set ON_ERROR_STOP TRUE
-\set postgis_version '3.1.5'
+\set postgis_version '3.1.4'
 
 CREATE EXTENSION IF NOT EXISTS postgis WITH VERSION :'postgis_version';
 ALTER EXTENSION postgis UPDATE TO :'postgis_version';

From 461eb2903df1f4f12e7c58b95a6641117923321a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:14:32 -0900
Subject: [PATCH 036/144] revert postgis version

---
 install-or-upgrade-postgis.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/install-or-upgrade-postgis.sql b/install-or-upgrade-postgis.sql
index 55f464f8..f701aa06 100644
--- a/install-or-upgrade-postgis.sql
+++ b/install-or-upgrade-postgis.sql
@@ -1,5 +1,5 @@
 \set ON_ERROR_STOP TRUE
-\set postgis_version '3.1.4'
+\set postgis_version '3.1.5'
 
 CREATE EXTENSION IF NOT EXISTS postgis WITH VERSION :'postgis_version';
 ALTER EXTENSION postgis UPDATE TO :'postgis_version';

From c5ebf7b63d70f753a4a8da0b928a1907f137cf5a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:30:50 -0900
Subject: [PATCH 037/144] update README

---
 README.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/README.md b/README.md
index d600f4aa..65fb0efe 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,18 @@ TODO: document creating conda env and installing developer deps
 
 ## STAC API
 
+### Upgrading the database
+
+TODO: document upgrading Postgres, PostGIS, and PgSTAC
+
+Run the following command to list the Postgres versions supported by Amazon RDS:
+
+```
+aws rds describe-db-engine-versions --engine postgres --query "DBEngineVersions[].EngineVersion"
+```
+
+To upgrade to a newer version, change the version specified in <apps/database/cloudformation.yml>.
+
 ### Connecting to the database
 
 Confirm you have the `psql` command installed, then run:

From f21a34995969d09cadea8ce401dc5853e04e51d8 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:32:11 -0900
Subject: [PATCH 038/144] update README

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 65fb0efe..efd43a06 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,8 @@ Run the following command to list the Postgres versions supported by Amazon RDS:
 aws rds describe-db-engine-versions --engine postgres --query "DBEngineVersions[].EngineVersion"
 ```
 
-To upgrade to a newer version, change the version specified in <apps/database/cloudformation.yml>.
+To upgrade to a newer version, change the version specified in the
+[database CloudFormation template](apps/database/cloudformation.yml).
 
 ### Connecting to the database
 

From 53695543ce1e8ec62bcc84ad3660cf61396c1ba7 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:41:13 -0900
Subject: [PATCH 039/144] update README

---
 README.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index efd43a06..bd09454e 100644
--- a/README.md
+++ b/README.md
@@ -18,9 +18,20 @@ Run the following command to list the Postgres versions supported by Amazon RDS:
 aws rds describe-db-engine-versions --engine postgres --query "DBEngineVersions[].EngineVersion"
 ```
 
-To upgrade to a newer version, change the version specified in the
+To upgrade Postgres, change the version specified in the
 [database CloudFormation template](apps/database/cloudformation.yml).
 
+When you upgrade Postgres, you should also upgrade the PostGIS extension. Refer to the tables shown
+[here](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html)
+to determine which version of PostGIS is supported by the database's Postgres version. You can also connect to the
+database and run the following query:
+
+```
+SELECT * FROM pg_available_extension_versions WHERE name='postgis';
+```
+
+To upgrade PostGIS, change the version specified in the [install/upgrade script](install-or-upgrade-postgis.sql).
+
 ### Connecting to the database
 
 Confirm you have the `psql` command installed, then run:

From 934e446e48c9ef4f3206f3b9441a9087794d8be9 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:43:21 -0900
Subject: [PATCH 040/144] try to simplify database migration requirements

---
 requirements-database-migrations.txt | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/requirements-database-migrations.txt b/requirements-database-migrations.txt
index 7165dc9c..d267e114 100644
--- a/requirements-database-migrations.txt
+++ b/requirements-database-migrations.txt
@@ -1,4 +1 @@
-psycopg==3.0.18
-psycopg-binary==3.0.18
-psycopg-pool==3.1.4
-pypgstac==0.6.10
+pypgstac[psycopg]==0.6.10

From 09f189a7434c0b16207695eef05c80391bf8d03b Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:54:57 -0900
Subject: [PATCH 041/144] Make pypgstac migration more obvious

---
 .github/workflows/deploy-stac-api.yml |  8 ++++----
 Makefile                              | 10 +++++-----
 requirements-database-migrations.txt  |  1 -
 requirements.txt                      |  1 +
 4 files changed, 10 insertions(+), 10 deletions(-)
 delete mode 100644 requirements-database-migrations.txt

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 4ca4ec4d..29b5428f 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -29,8 +29,8 @@ jobs:
           python-version: 3.9
       - name: Install Lambda dependencies
         run: make -s install-lambda-deps
-      - name: Install database migration dependencies
-        run: make -s install-db-migration-deps
+      - name: Install pyPgSTAC
+        run: make -s install-pypgstac
       - name: Deploy CloudFormation stack
         run: |
           make -s deploy \
@@ -46,5 +46,5 @@ jobs:
                                 --output text)" >> $GITHUB_ENV
       - name: Install or upgrade PostGIS
         run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
-      - name: Migrate database
-        run: make -s migrate db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
+      - name: Install or upgrade PgSTAC
+        run: make -s pypgstac-migrate db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
diff --git a/Makefile b/Makefile
index 7d2c38aa..47662442 100644
--- a/Makefile
+++ b/Makefile
@@ -2,13 +2,13 @@ install:
 	python -m pip install --upgrade pip && \
 	python -m pip install -r requirements.txt
 
-install-lambda-deps:
+install-pypgstac:
 	python -m pip install --upgrade pip && \
-	python -m pip install -r apps/api/requirements.txt -t apps/api/src/
+	python -m pip install $$(grep pypgstac requirements.txt)
 
-install-db-migration-deps:
+install-lambda-deps:
 	python -m pip install --upgrade pip && \
-	python -m pip install -r requirements-database-migrations.txt
+	python -m pip install -r apps/api/requirements.txt -t apps/api/src/
 
 s3_bucket ?= ''
 stack_name ?= ''
@@ -36,7 +36,7 @@ psql:
 install-or-upgrade-postgis:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql -f install-or-upgrade-postgis.sql
 
-migrate:
+pypgstac-migrate:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} pypgstac migrate
 
 run-api:
diff --git a/requirements-database-migrations.txt b/requirements-database-migrations.txt
deleted file mode 100644
index d267e114..00000000
--- a/requirements-database-migrations.txt
+++ /dev/null
@@ -1 +0,0 @@
-pypgstac[psycopg]==0.6.10
diff --git a/requirements.txt b/requirements.txt
index 6a997d5c..95da1328 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,6 +3,7 @@
 boto3==1.26.3
 cfn-lint==0.71.0
 flake8==5.0.4
+pypgstac[psycopg]==0.6.10
 pystac==1.6.1
 requests==2.28.1
 shapely==1.8.5.post1

From 3117f5bd6ec85da2bdbdd096438daeffde0e5958 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 10:55:52 -0900
Subject: [PATCH 042/144] remove unused requirement line

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 95da1328..0e9b1b7b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
--r requirements-database-migrations.txt
 -r apps/api/requirements.txt
 boto3==1.26.3
 cfn-lint==0.71.0

From 6b2942805777af59857d379661db6764c0ea5918 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 11:03:30 -0900
Subject: [PATCH 043/144] finish documenting database upgrades in README

---
 README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index bd09454e..60392c63 100644
--- a/README.md
+++ b/README.md
@@ -10,8 +10,6 @@ TODO: document creating conda env and installing developer deps
 
 ### Upgrading the database
 
-TODO: document upgrading Postgres, PostGIS, and PgSTAC
-
 Run the following command to list the Postgres versions supported by Amazon RDS:
 
 ```
@@ -32,6 +30,10 @@ SELECT * FROM pg_available_extension_versions WHERE name='postgis';
 
 To upgrade PostGIS, change the version specified in the [install/upgrade script](install-or-upgrade-postgis.sql).
 
+PgSTAC upgrades are handled automatically: the [deployment workflow](.github/workflows/deploy-stac-api.yml)
+migrates the database to the installed version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac>
+for more information about migrations.
+
 ### Connecting to the database
 
 Confirm you have the `psql` command installed, then run:

From 9ab96dd6ce67c2563969c12778527b138370eb47 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 11:11:45 -0900
Subject: [PATCH 044/144] update README

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 60392c63..d22b75a7 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,9 @@ TODO: document creating conda env and installing developer deps
 
 ### Upgrading the database
 
+Our deployment workflow creates a Postgres database, installs the PostGIS extension, and then installs
+[PgSTAC](https://stac-utils.github.io/pgstac). Below are instructions for upgrading the database.
+
 Run the following command to list the Postgres versions supported by Amazon RDS:
 
 ```

From d948b6882a9a4f42968e06765c4c9382a3faf50d Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 12:24:55 -0900
Subject: [PATCH 045/144] update README database upgrade steps

---
 README.md | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/README.md b/README.md
index d22b75a7..704233b7 100644
--- a/README.md
+++ b/README.md
@@ -10,28 +10,27 @@ TODO: document creating conda env and installing developer deps
 
 ### Upgrading the database
 
-Our deployment workflow creates a Postgres database, installs the PostGIS extension, and then installs
-[PgSTAC](https://stac-utils.github.io/pgstac). Below are instructions for upgrading the database.
+The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs
+[PgSTAC](https://stac-utils.github.io/pgstac). Follow these steps to upgrade the database:
 
-Run the following command to list the Postgres versions supported by Amazon RDS:
+1. Run the following command to list the Postgres versions supported by Amazon RDS:
+    ```
+    aws rds describe-db-engine-versions --engine postgres
+    ```
+   Identify the entry that corresponds to the current version of the database.
+   Then identify the newest available version from the list of valid upgrade targets given for the current version.
+   This will be the new version for the database.
 
-```
-aws rds describe-db-engine-versions --engine postgres --query "DBEngineVersions[].EngineVersion"
-```
-
-To upgrade Postgres, change the version specified in the
-[database CloudFormation template](apps/database/cloudformation.yml).
+2. Change the Postgres version specified in the [database CloudFormation template](apps/database/cloudformation.yml)
+   to the new version.
 
-When you upgrade Postgres, you should also upgrade the PostGIS extension. Refer to the tables shown
-[here](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html)
-to determine which version of PostGIS is supported by the database's Postgres version. You can also connect to the
-database and run the following query:
+3. Next, refer to the tables shown
+   [here](https://docs.aws.amazon.com/AmazonRDS/latest/PostgreSQLReleaseNotes/postgresql-extensions.html)
+   to determine which version of the PostGIS extension is supported by the new Postgres version.
 
-```
-SELECT * FROM pg_available_extension_versions WHERE name='postgis';
-```
+4. Change the PostGIS version specified in the [install/upgrade script](install-or-upgrade-postgis.sql).
 
-To upgrade PostGIS, change the version specified in the [install/upgrade script](install-or-upgrade-postgis.sql).
+5. Deploy to AWS.
 
 PgSTAC upgrades are handled automatically: the [deployment workflow](.github/workflows/deploy-stac-api.yml)
 migrates the database to the installed version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac>

From 321d4b6e80a97eb220101ec56391231bdb6b3b1a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 12:28:56 -0900
Subject: [PATCH 046/144] update README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 704233b7..5a68630b 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ The initial AWS deployment creates a Postgres database, installs the PostGIS ext
     aws rds describe-db-engine-versions --engine postgres
     ```
    Identify the entry that corresponds to the current version of the database.
-   Then identify the newest available version from the list of valid upgrade targets given for the current version.
+   Then identify the newest available version from the list of valid upgrade targets for the current version.
    This will be the new version for the database.
 
 2. Change the Postgres version specified in the [database CloudFormation template](apps/database/cloudformation.yml)

From 8639bbba0a84c565ec46a09a3a1776e721bad0a5 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 15:50:16 -0900
Subject: [PATCH 047/144] Increase lambda memory/cpu

---
 apps/api/cloudformation.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 4878be22..06a68da4 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -23,6 +23,7 @@ Resources:
           POSTGRES_PASS: !Ref DatabasePassword
       Code: src/
       Handler: api.handler
+      MemorySize: 1000
       Role: !GetAtt LambdaRole.Arn
       Runtime: python3.9
       Timeout: 30

From c5adf8f2436959e13d75c31686b8e1f9bf22e266 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 15:56:37 -0900
Subject: [PATCH 048/144] increase lambda memory/cpu to 2gb

---
 apps/api/cloudformation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 06a68da4..f2c14754 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -23,7 +23,7 @@ Resources:
           POSTGRES_PASS: !Ref DatabasePassword
       Code: src/
       Handler: api.handler
-      MemorySize: 1000
+      MemorySize: 2000
       Role: !GetAtt LambdaRole.Arn
       Runtime: python3.9
       Timeout: 30

From 8bade9685bc947d61df08ddea0f126f3d4eac902 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 16:03:00 -0900
Subject: [PATCH 049/144] increase lambda memory to 4 gb

---
 apps/api/cloudformation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index f2c14754..cadf485c 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -23,7 +23,7 @@ Resources:
           POSTGRES_PASS: !Ref DatabasePassword
       Code: src/
       Handler: api.handler
-      MemorySize: 2000
+      MemorySize: 4000
       Role: !GetAtt LambdaRole.Arn
       Runtime: python3.9
       Timeout: 30

From 25105b7b5ec5b097ca319314bc3d16d3498a7989 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 9 Nov 2022 16:09:51 -0900
Subject: [PATCH 050/144] decrease lambda memory back to 2gb

---
 apps/api/cloudformation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index cadf485c..f2c14754 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -23,7 +23,7 @@ Resources:
           POSTGRES_PASS: !Ref DatabasePassword
       Code: src/
       Handler: api.handler
-      MemorySize: 4000
+      MemorySize: 2000
       Role: !GetAtt LambdaRole.Arn
       Runtime: python3.9
       Timeout: 30

From f85f7791fbd2b3ff706a6e3c68044d5f47c21baa Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 10 Nov 2022 11:24:42 -0900
Subject: [PATCH 051/144] Use 3 db roles (admin, ingest, read)

---
 .github/workflows/deploy-stac-api.yml | 17 +++++++++++++----
 Makefile                              | 26 ++++++++++++++------------
 README.md                             |  2 +-
 apps/api/cloudformation.yml           |  6 +++---
 apps/cloudformation.yml               | 10 +++++++---
 apps/database/cloudformation.yml      |  4 ++--
 configure-database-roles.sql          |  4 ++++
 7 files changed, 44 insertions(+), 25 deletions(-)
 create mode 100644 configure-database-roles.sql

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 29b5428f..1b213830 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -10,7 +10,9 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   CF_TEMPLATES_BUCKET: ${{ secrets.CF_TEMPLATES_BUCKET }}
   CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-  DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
+  DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
+  DB_INGEST_PASSWORD: ${{ secrets.DB_INGEST_PASSWORD }}
+  DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
   STACK_NAME: stac-api-sandbox
 
 jobs:
@@ -37,7 +39,7 @@ jobs:
             s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
             stack_name=${{ env.STACK_NAME }} \
             cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
-            new_db_password=${{ env.DATABASE_PASSWORD }}
+            db_admin_password=${{ env.DB_ADMIN_PASSWORD }}
       - name: Get database host
         run: |
           echo "DATABASE_HOST=$(aws cloudformation describe-stacks \
@@ -45,6 +47,13 @@ jobs:
                                 --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
                                 --output text)" >> $GITHUB_ENV
       - name: Install or upgrade PostGIS
-        run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
+        run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_admin_password=${{ env.DB_ADMIN_PASSWORD }}
       - name: Install or upgrade PgSTAC
-        run: make -s pypgstac-migrate db_host=$DATABASE_HOST db_password=${{ env.DATABASE_PASSWORD }}
+        run: make -s pypgstac-migrate db_host=$DATABASE_HOST db_admin_password=${{ env.DB_ADMIN_PASSWORD }}
+      - name: Configure database roles
+        run: |
+          make -s configure-database-roles \
+            db_host=$DATABASE_HOST \
+            db_admin_password=${{ env.DB_ADMIN_PASSWORD }} \
+            db_ingest_password=${{ env.DB_INGEST_PASSWORD }} \
+            db_read_password=${{ env.DB_READ_PASSWORD }}
diff --git a/Makefile b/Makefile
index 47662442..8ab64e2f 100644
--- a/Makefile
+++ b/Makefile
@@ -10,11 +10,6 @@ install-lambda-deps:
 	python -m pip install --upgrade pip && \
 	python -m pip install -r apps/api/requirements.txt -t apps/api/src/
 
-s3_bucket ?= ''
-stack_name ?= ''
-cloudformation_role_arn ?= ''
-new_db_password ?= ''
-# TODO allow adding --role-arn option to deploy command, for cicd
 deploy:
 	aws cloudformation package \
 	    --template-file apps/cloudformation.yml \
@@ -26,21 +21,28 @@ deploy:
 	    --capabilities CAPABILITY_NAMED_IAM \
 	    --role-arn ${cloudformation_role_arn} \
 	    --parameter-overrides \
-	      DatabasePassword=${new_db_password}
+	      DatabasePassword=${db_admin_password}
 
-db_host ?= ''
-db_password ?= ''
 psql:
-	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql
 
 install-or-upgrade-postgis:
-	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} psql -f install-or-upgrade-postgis.sql
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql \
+	    -f install-or-upgrade-postgis.sql
+
+configure-database-roles:
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql \
+	    --set=db_ingest_password=${db_ingest_password} \
+	    --set=db_read_password=${db_read_password} \
+	    -f configure-database-roles.sql
 
 pypgstac-migrate:
-	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_password} pypgstac migrate
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} pypgstac migrate
 
 run-api:
-	POSTGRES_HOST_READER=${db_host} POSTGRES_HOST_WRITER=${db_host} POSTGRES_PORT=5432 POSTGRES_DBNAME=postgres POSTGRES_USER=postgres POSTGRES_PASS=${db_password} python -m stac_fastapi.pgstac.app
+	POSTGRES_HOST_READER=${db_host} POSTGRES_HOST_WRITER=${db_host} POSTGRES_PORT=5432 \
+	    POSTGRES_DBNAME=postgres POSTGRES_USER=pgstac_ingest POSTGRES_PASS=${db_ingest_password} \
+	    python -m stac_fastapi.pgstac.app
 
 static: flake8 cfn-lint
 
diff --git a/README.md b/README.md
index 5a68630b..369571ff 100644
--- a/README.md
+++ b/README.md
@@ -52,7 +52,7 @@ RDS console and copying the "Endpoint" field.
 To run the STAC API locally:
 
 ```
-make run-api db_host=<host> db_password=<password>
+make run-api db_host=<host> db_ingest_password=<password>
 ```
 
 You should see something like `Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)` in the output; you can
diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index f2c14754..5d44f9b4 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -5,7 +5,7 @@ Parameters:
     Type: String
     NoEcho: true
 
-  DatabasePassword:
+  DatabaseReadPassword:
     Type: String
     NoEcho: true
 
@@ -19,8 +19,8 @@ Resources:
           POSTGRES_HOST_WRITER: !Ref DatabaseHost
           POSTGRES_PORT: 5432
           POSTGRES_DBNAME: postgres
-          POSTGRES_USER: postgres
-          POSTGRES_PASS: !Ref DatabasePassword
+          POSTGRES_USER: pgstac_read
+          POSTGRES_PASS: !Ref DatabaseReadPassword
       Code: src/
       Handler: api.handler
       MemorySize: 2000
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 69a43380..754018ec 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -1,7 +1,11 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
-  DatabasePassword:
+  DatabaseAdminPassword:
+    Type: String
+    NoEcho: true
+
+  DatabaseReadPassword:
     Type: String
     NoEcho: true
 
@@ -15,12 +19,12 @@ Resources:
     Properties:
       Parameters:
         DatabaseHost: !GetAtt Database.Outputs.DatabaseHost
-        DatabasePassword: !Ref DatabasePassword
+        DatabaseReadPassword: !Ref DatabaseReadPassword
       TemplateURL: api/cloudformation.yml
 
   Database:
     Type: AWS::CloudFormation::Stack
     Properties:
       Parameters:
-        DatabasePassword: !Ref DatabasePassword
+        DatabaseAdminPassword: !Ref DatabaseAdminPassword
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index abf532a1..a493557a 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -1,7 +1,7 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
-  DatabasePassword:
+  DatabaseAdminPassword:
     Type: String
     NoEcho: true
 
@@ -20,7 +20,7 @@ Resources:
       Engine: postgres
       EngineVersion: '14.4'
       MasterUsername: postgres
-      MasterUserPassword: !Ref DatabasePassword
+      MasterUserPassword: !Ref DatabaseAdminPassword
       # TODO: MaxAllocatedStorage: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-maxallocatedstorage
       Port: '5432'
       # TODO: ProcessorFeatures: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-processorfeatures
diff --git a/configure-database-roles.sql b/configure-database-roles.sql
new file mode 100644
index 00000000..fb491e2f
--- /dev/null
+++ b/configure-database-roles.sql
@@ -0,0 +1,4 @@
+\set ON_ERROR_STOP TRUE
+
+ALTER ROLE pgstac_ingest LOGIN PASSWORD :'db_ingest_password';
+ALTER ROLE pgstac_read LOGIN PASSWORD :'db_read_password';

From e4f7b7c1b83a29ec3f5b16fa298c9747e288bc4a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 10 Nov 2022 11:36:46 -0900
Subject: [PATCH 052/144] Add missing cf parameter

---
 .github/workflows/deploy-stac-api.yml | 3 ++-
 Makefile                              | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 1b213830..3cb197bd 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -39,7 +39,8 @@ jobs:
             s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
             stack_name=${{ env.STACK_NAME }} \
             cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
-            db_admin_password=${{ env.DB_ADMIN_PASSWORD }}
+            db_admin_password=${{ env.DB_ADMIN_PASSWORD }} \
+            db_read_password=${{ env.DB_READ_PASSWORD }}
       - name: Get database host
         run: |
           echo "DATABASE_HOST=$(aws cloudformation describe-stacks \
diff --git a/Makefile b/Makefile
index 8ab64e2f..b967a018 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,8 @@ deploy:
 	    --capabilities CAPABILITY_NAMED_IAM \
 	    --role-arn ${cloudformation_role_arn} \
 	    --parameter-overrides \
-	      DatabasePassword=${db_admin_password}
+	      DatabaseAdminPassword=${db_admin_password} \
+	      DatabaseReadPassword=${db_read_password}
 
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql

From 538a5adfa3318be2912272a6d5a626cc8580e310 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 10 Nov 2022 12:06:31 -0900
Subject: [PATCH 053/144] update make psql target

---
 Makefile  | 2 +-
 README.md | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index b967a018..a9c27a05 100644
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,7 @@ deploy:
 	      DatabaseReadPassword=${db_read_password}
 
 psql:
-	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=${db_user} PGPASSWORD=${db_password} psql
 
 install-or-upgrade-postgis:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql \
diff --git a/README.md b/README.md
index 369571ff..89703d29 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,8 @@ TODO: document creating conda env and installing developer deps
 
 ## STAC API
 
+TODO: proofread docs since adding three database users
+
 ### Upgrading the database
 
 The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs
@@ -41,7 +43,7 @@ for more information about migrations.
 Confirm you have the `psql` command installed, then run:
 
 ```
-make psql db_host=<host> db_password=<password>
+make psql db_host=<host> db_user=<user> db_password=<password>
 ```
 
 You can find the appropriate value for `<host>` by navigating to the database instance via the CloudFormation or

From 970b977b4c87e8fd6a724ce133e5249a45181e65 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 10 Nov 2022 15:58:36 -0900
Subject: [PATCH 054/144] fix database perms bug

---
 configure-database-roles.sql | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/configure-database-roles.sql b/configure-database-roles.sql
index fb491e2f..93679ebb 100644
--- a/configure-database-roles.sql
+++ b/configure-database-roles.sql
@@ -2,3 +2,6 @@
 
 ALTER ROLE pgstac_ingest LOGIN PASSWORD :'db_ingest_password';
 ALTER ROLE pgstac_read LOGIN PASSWORD :'db_read_password';
+
+-- TODO this will no longer be needed after the pgstac_read permissions bug is fixed: https://github.com/stac-utils/pgstac/issues/146
+GRANT SELECT ON pgstac.collections TO pgstac_read;

From 1e0bb071c2391327b7e9f54aa1322d25961c55ca Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 10 Nov 2022 17:02:35 -0900
Subject: [PATCH 055/144] do not use pgstac_ingest user

---
 .github/workflows/deploy-stac-api.yml | 1 -
 Makefile                              | 3 +--
 README.md                             | 2 +-
 configure-database-roles.sql          | 3 ---
 4 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 3cb197bd..6aa465a9 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -56,5 +56,4 @@ jobs:
           make -s configure-database-roles \
             db_host=$DATABASE_HOST \
             db_admin_password=${{ env.DB_ADMIN_PASSWORD }} \
-            db_ingest_password=${{ env.DB_INGEST_PASSWORD }} \
             db_read_password=${{ env.DB_READ_PASSWORD }}
diff --git a/Makefile b/Makefile
index a9c27a05..9dadef16 100644
--- a/Makefile
+++ b/Makefile
@@ -33,7 +33,6 @@ install-or-upgrade-postgis:
 
 configure-database-roles:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql \
-	    --set=db_ingest_password=${db_ingest_password} \
 	    --set=db_read_password=${db_read_password} \
 	    -f configure-database-roles.sql
 
@@ -42,7 +41,7 @@ pypgstac-migrate:
 
 run-api:
 	POSTGRES_HOST_READER=${db_host} POSTGRES_HOST_WRITER=${db_host} POSTGRES_PORT=5432 \
-	    POSTGRES_DBNAME=postgres POSTGRES_USER=pgstac_ingest POSTGRES_PASS=${db_ingest_password} \
+	    POSTGRES_DBNAME=postgres POSTGRES_USER=postgres POSTGRES_PASS=${db_admin_password} \
 	    python -m stac_fastapi.pgstac.app
 
 static: flake8 cfn-lint
diff --git a/README.md b/README.md
index 89703d29..929e3b64 100644
--- a/README.md
+++ b/README.md
@@ -54,7 +54,7 @@ RDS console and copying the "Endpoint" field.
 To run the STAC API locally:
 
 ```
-make run-api db_host=<host> db_ingest_password=<password>
+make run-api db_host=<host> db_admin_password=<password>
 ```
 
 You should see something like `Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)` in the output; you can
diff --git a/configure-database-roles.sql b/configure-database-roles.sql
index 93679ebb..f2eb6981 100644
--- a/configure-database-roles.sql
+++ b/configure-database-roles.sql
@@ -1,7 +1,4 @@
 \set ON_ERROR_STOP TRUE
 
-ALTER ROLE pgstac_ingest LOGIN PASSWORD :'db_ingest_password';
 ALTER ROLE pgstac_read LOGIN PASSWORD :'db_read_password';
-
--- TODO this will no longer be needed after the pgstac_read permissions bug is fixed: https://github.com/stac-utils/pgstac/issues/146
 GRANT SELECT ON pgstac.collections TO pgstac_read;

From 11f9e9fd6943d601367501ad42c4cd84ac0d925e Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 10 Nov 2022 17:20:04 -0900
Subject: [PATCH 056/144] remove unused deploy env var

---
 .github/workflows/deploy-stac-api.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 6aa465a9..6ec11ea5 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -11,7 +11,6 @@ env:
   CF_TEMPLATES_BUCKET: ${{ secrets.CF_TEMPLATES_BUCKET }}
   CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
   DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
-  DB_INGEST_PASSWORD: ${{ secrets.DB_INGEST_PASSWORD }}
   DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
   STACK_NAME: stac-api-sandbox
 

From 7b3388ece8a55c6606b513bf9ae78a08227405d9 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 09:37:37 -0900
Subject: [PATCH 057/144] document Transaction extension in README

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 929e3b64..f9fc77f7 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,10 @@ RDS console and copying the "Endpoint" field.
 
 ### Running the API locally
 
+You can run the STAC API frontend locally (connected to the AWS-hosted database). This is required for accessing
+the create/update/delete endpoints (which are provided by the STAC API's Transaction extension), as these
+endpoints are disabled for the publicly accessible API.
+
 To run the STAC API locally:
 
 ```
@@ -62,4 +66,5 @@ query the API at that URL.
 
 ### Ingesting STAC dataset
 
-Run `python ingest_data.py -h` for usage instructions.
+Run `python ingest_data.py -h` for usage instructions. The script only works with an API that supports the
+Transaction extension.

From a515184d3fd0bb98a5dbbff9d707d7141dac0f1b Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 09:54:38 -0900
Subject: [PATCH 058/144] add cfn-lint github action

---
 .github/workflows/static-analysis.yml | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 .github/workflows/static-analysis.yml

diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml
new file mode 100644
index 00000000..cde098da
--- /dev/null
+++ b/.github/workflows/static-analysis.yml
@@ -0,0 +1,14 @@
+name: Static code analysis
+
+on: push
+
+jobs:
+  cfn-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3.1.0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - run: make install
+      - run: make cfn-lint

From 17fc3317a8003d12e4b6c269e3eb17a89f884043 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 10:55:01 -0900
Subject: [PATCH 059/144] deploy db passwords to secrets manager

---
 apps/cloudformation.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 754018ec..b5b29e50 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -28,3 +28,15 @@ Resources:
       Parameters:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
       TemplateURL: database/cloudformation.yml
+
+  DatabaseAdminSecret:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: !Sub "${AWS::StackName} database credentials for 'postgres' user"
+      SecretString: !Sub '"password": "${DatabaseAdminPassword}"}'
+
+  DatabaseReadSecret:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: !Sub "${AWS::StackName} database credentials for 'pgstac_read' user"
+      SecretString: !Sub '"password": "${DatabaseReadPassword}"}'

From f33b3ebbac78b9b04ad597b516a523f48870d3e6 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 11:07:14 -0900
Subject: [PATCH 060/144] fix secret strings

---
 apps/cloudformation.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index b5b29e50..c6978b51 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -33,10 +33,10 @@ Resources:
     Type: AWS::SecretsManager::Secret
     Properties:
       Description: !Sub "${AWS::StackName} database credentials for 'postgres' user"
-      SecretString: !Sub '"password": "${DatabaseAdminPassword}"}'
+      SecretString: !Sub '{"password": "${DatabaseAdminPassword}"}'
 
   DatabaseReadSecret:
     Type: AWS::SecretsManager::Secret
     Properties:
       Description: !Sub "${AWS::StackName} database credentials for 'pgstac_read' user"
-      SecretString: !Sub '"password": "${DatabaseReadPassword}"}'
+      SecretString: !Sub '{"password": "${DatabaseReadPassword}"}'

From 670def1abb1e3a2d5e274e16223ad4a5e2e3191e Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 11:13:37 -0900
Subject: [PATCH 061/144] simplify database secrets

---
 apps/cloudformation.yml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index c6978b51..05777f58 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -29,14 +29,8 @@ Resources:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
       TemplateURL: database/cloudformation.yml
 
-  DatabaseAdminSecret:
+  DatabaseSecret:
     Type: AWS::SecretsManager::Secret
     Properties:
-      Description: !Sub "${AWS::StackName} database credentials for 'postgres' user"
-      SecretString: !Sub '{"password": "${DatabaseAdminPassword}"}'
-
-  DatabaseReadSecret:
-    Type: AWS::SecretsManager::Secret
-    Properties:
-      Description: !Sub "${AWS::StackName} database credentials for 'pgstac_read' user"
-      SecretString: !Sub '{"password": "${DatabaseReadPassword}"}'
+      Description: !Sub "${AWS::StackName} database credentials"
+      SecretString: !Sub '{"admin_user": "postgres", "admin_password": "${DatabaseAdminPassword}", "read_user": "pgstac_read", "read_password": "${DatabaseReadPassword}"}'

From db8fb80d8dfe74e0b3a415adca55fd3a65ec24d0 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 11:30:17 -0900
Subject: [PATCH 062/144] Add db host to secret

---
 README.md               | 8 +++++---
 apps/cloudformation.yml | 2 +-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index f9fc77f7..52a63e82 100644
--- a/README.md
+++ b/README.md
@@ -38,6 +38,11 @@ PgSTAC upgrades are handled automatically: the [deployment workflow](.github/wor
 migrates the database to the installed version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac>
 for more information about migrations.
 
+### Retrieving database connection details
+
+The database host and database user credentials are available via the AWS Secrets Manager console
+in the AWS account where the CloudFormation stack was deployed.
+
 ### Connecting to the database
 
 Confirm you have the `psql` command installed, then run:
@@ -46,9 +51,6 @@ Confirm you have the `psql` command installed, then run:
 make psql db_host=<host> db_user=<user> db_password=<password>
 ```
 
-You can find the appropriate value for `<host>` by navigating to the database instance via the CloudFormation or
-RDS console and copying the "Endpoint" field.
-
 ### Running the API locally
 
 You can run the STAC API frontend locally (connected to the AWS-hosted database). This is required for accessing
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 05777f58..55ba7d7a 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -33,4 +33,4 @@ Resources:
     Type: AWS::SecretsManager::Secret
     Properties:
       Description: !Sub "${AWS::StackName} database credentials"
-      SecretString: !Sub '{"admin_user": "postgres", "admin_password": "${DatabaseAdminPassword}", "read_user": "pgstac_read", "read_password": "${DatabaseReadPassword}"}'
+      SecretString: !Sub '{"database_host": "${Database.Outputs.DatabaseHost}", "admin_user": "postgres", "admin_password": "${DatabaseAdminPassword}", "read_user": "pgstac_read", "read_password": "${DatabaseReadPassword}"}'

From cdcbaaaee50fbc5b1a09755447b28e09f8ccfc2d Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 11:32:23 -0900
Subject: [PATCH 063/144] update readme

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 52a63e82..33e19d65 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ TODO: document creating conda env and installing developer deps
 
 ## STAC API
 
-TODO: proofread docs since adding three database users
+TODO: proofread docs since adding multiple database users
 
 ### Upgrading the database
 

From 7b5756420c43615e031907e4d1fb43e23e6eb1ad Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 12:17:29 -0900
Subject: [PATCH 064/144] limit network access to database

---
 apps/api/cloudformation.yml      | 13 ++++++++++++-
 apps/cloudformation.yml          | 13 +++++++++++++
 apps/database/cloudformation.yml | 21 ++++++++++++++++++---
 3 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 5d44f9b4..167bf61d 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -9,6 +9,12 @@ Parameters:
     Type: String
     NoEcho: true
 
+  SubnetIds:
+    Type: CommaDelimitedList
+
+  SecurityGroupId:
+    Type: AWS::EC2::SecurityGroup::Id
+
 Resources:
   Lambda:
     Type: AWS::Lambda::Function
@@ -27,6 +33,10 @@ Resources:
       Role: !GetAtt LambdaRole.Arn
       Runtime: python3.9
       Timeout: 30
+      VpcConfig:
+        SecurityGroupIds:
+          - !Ref SecurityGroupId
+        SubnetIds: !Ref SubnetIds
 
   LambdaLogGroup:
     Type: AWS::Logs::LogGroup
@@ -46,6 +56,7 @@ Resources:
           Effect: Allow
       ManagedPolicyArns:
         - !Ref LambdaPolicy
+        - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole
 
   LambdaPolicy:
     Type: AWS::IAM::ManagedPolicy
@@ -98,4 +109,4 @@ Resources:
             Statement:
               - Effect: Allow
                 Action: lambda:InvokeFunction
-                Resource: !GetAtt Lambda.Arn
\ No newline at end of file
+                Resource: !GetAtt Lambda.Arn
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 754018ec..99d6d8d1 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -1,6 +1,15 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+
+  SubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+
+  CidrIp:
+    Type: String
+
   DatabaseAdminPassword:
     Type: String
     NoEcho: true
@@ -20,6 +29,8 @@ Resources:
       Parameters:
         DatabaseHost: !GetAtt Database.Outputs.DatabaseHost
         DatabaseReadPassword: !Ref DatabaseReadPassword
+        SecurityGroupId: !GetAtt Database.Outputs.ClientSecurityGroupId
+        SubnetIds: !Join [",", !Ref SubnetIds]
       TemplateURL: api/cloudformation.yml
 
   Database:
@@ -27,4 +38,6 @@ Resources:
     Properties:
       Parameters:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
+        VpcId: !Ref VpcId
+        CidrIp: !Ref CidrIp
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index a493557a..4421f5d4 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -5,10 +5,19 @@ Parameters:
     Type: String
     NoEcho: true
 
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+
+  CidrIp:
+    Type: String
+
 Outputs:
   DatabaseHost:
     Value: !GetAtt DatabaseInstance.Endpoint.Address
 
+  ClientSecurityGroupId:
+    Value: !Ref ClientSecurityGroup
+
 Resources:
   DatabaseInstance:
     Type: AWS::RDS::DBInstance
@@ -27,10 +36,16 @@ Resources:
       PubliclyAccessible: true
       # TODO: StorageType: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-storagetype
 
-  # TODO only allow incoming connections from within the ASF network and from our Lambda function
   DatabaseSecurityGroup:
     Type: AWS::RDS::DBSecurityGroup
     Properties:
+      GroupDescription: !Sub "Security group for ${AWS::StackName} database"
       DBSecurityGroupIngress:
-        - CIDRIP: 0.0.0.0/0
-      GroupDescription: STAC API database security group.
+        - EC2SecurityGroupId: !Ref ClientSecurityGroup
+        - CIDRIP: !Ref CidrIp
+
+  ClientSecurityGroup:
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: !Sub "Security group for clients of ${AWS::StackName} database"
+      VpcId: !Ref VpcId

From a952a0cb9ee0907e3f50998633a219678c9e48ed Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 12:20:24 -0900
Subject: [PATCH 065/144] add new stack parameters to makefile and actions

---
 .github/workflows/deploy-stac-api.yml | 9 ++++++++-
 Makefile                              | 5 ++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 6ec11ea5..17bf034e 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -13,6 +13,9 @@ env:
   DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
   DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
   STACK_NAME: stac-api-sandbox
+  VPD_ID: ${{ secrets.VPD_ID }}
+  SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
+  CIDR_IP: ${{ secrets.CIDR_IP }}
 
 jobs:
   deploy-stac-api:
@@ -39,7 +42,11 @@ jobs:
             stack_name=${{ env.STACK_NAME }} \
             cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
             db_admin_password=${{ env.DB_ADMIN_PASSWORD }} \
-            db_read_password=${{ env.DB_READ_PASSWORD }}
+            db_read_password=${{ env.DB_READ_PASSWORD }} \
+            vpc_id=${{ env.VPC_ID }} \
+            subnet_ids=${{ env.SUBNET_IDS }} \
+            cidr_ip=${{ env.CIDR_IP }}
+
       - name: Get database host
         run: |
           echo "DATABASE_HOST=$(aws cloudformation describe-stacks \
diff --git a/Makefile b/Makefile
index 9dadef16..7f9a515a 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,10 @@ deploy:
 	    --role-arn ${cloudformation_role_arn} \
 	    --parameter-overrides \
 	      DatabaseAdminPassword=${db_admin_password} \
-	      DatabaseReadPassword=${db_read_password}
+	      DatabaseReadPassword=${db_read_password} \
+	      VpcId=${vpc_id} \
+	      SubnetIds=${subnet_ids} \
+	      CidrIp=${cidr_ip}
 
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=${db_user} PGPASSWORD=${db_password} psql

From 522ba9e05712d0b199095a3ca77ed177ecdbebac Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 13:03:15 -0900
Subject: [PATCH 066/144] add labaled-pr github action

---
 .github/workflows/labeled-pr.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 .github/workflows/labeled-pr.yml

diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml
new file mode 100644
index 00000000..143d166b
--- /dev/null
+++ b/.github/workflows/labeled-pr.yml
@@ -0,0 +1,15 @@
+name: Is PR labeled?
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - unlabeled
+      - synchronize
+    branches:
+      - main
+
+jobs:
+  call-labeled-pr-check-workflow:
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.4.0
\ No newline at end of file

From 098bd03ea307f2fe0af131f3e2aad7881ce47f1d Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 13:03:40 -0900
Subject: [PATCH 067/144] add missing newline at end of file

---
 .github/workflows/labeled-pr.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/labeled-pr.yml b/.github/workflows/labeled-pr.yml
index 143d166b..a0a3ecf8 100644
--- a/.github/workflows/labeled-pr.yml
+++ b/.github/workflows/labeled-pr.yml
@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.4.0
\ No newline at end of file
+    uses: ASFHyP3/actions/.github/workflows/reusable-labeled-pr-check.yml@v0.4.0

From a7d11251a2f3ecd7a4c7ebc311a134a31d08b1ca Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 13:08:37 -0900
Subject: [PATCH 068/144] add changelog

---
 CHANGELOG.md | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 CHANGELOG.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000..89f215ff
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,9 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.0.0]
+### Added
+- Initial release of STAC API endpoint backed by a PostgreSQL database

From a9b878cd9dbadcee78aa318226a451d9d909caf8 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 13:10:07 -0900
Subject: [PATCH 069/144] add codeowners file

---
 .github/CODEOWNERS | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 .github/CODEOWNERS

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
new file mode 100644
index 00000000..3365cbe1
--- /dev/null
+++ b/.github/CODEOWNERS
@@ -0,0 +1,2 @@
+# These owners will be requested for review when someone opens a pull request.
+*       @ASFHyP3/platform

From 2561d4034e010915e6c3a217b70450114119ac23 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 13:22:16 -0900
Subject: [PATCH 070/144] add LICENCE file

---
 LICENCE | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 LICENCE

diff --git a/LICENCE b/LICENCE
new file mode 100644
index 00000000..4d2aabc7
--- /dev/null
+++ b/LICENCE
@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2022, Alaska Satellite Facility
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

From acbd83757a245af437ce38b68a9b79bf17612fda Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 13:26:12 -0900
Subject: [PATCH 071/144] add dependabot config

---
 .github/dependabot.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 .github/dependabot.yml

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 00000000..383b1536
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+      - "bumpless"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    labels:
+      - "bumpless"

From 18bacc51b3ecf9ec01af3ea8d4f84fdc7a1cabff Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 14:05:35 -0900
Subject: [PATCH 072/144] fix typo in deploy-stac-api.yml

Co-authored-by: Jake Herrmann <jtherrmann@alaska.edu>
---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 17bf034e..fb195771 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -13,7 +13,7 @@ env:
   DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
   DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
   STACK_NAME: stac-api-sandbox
-  VPD_ID: ${{ secrets.VPD_ID }}
+  VPC_ID: ${{ secrets.VPC_ID }}
   SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
   CIDR_IP: ${{ secrets.CIDR_IP }}
 

From 2c9429b65894dd6a39e8b4ce76d0730c016d6567 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 14:59:27 -0900
Subject: [PATCH 073/144] move api requirements to project root

---
 Makefile                                               | 2 +-
 apps/api/requirements.txt => requirements-apps-api.txt | 0
 requirements.txt                                       | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename apps/api/requirements.txt => requirements-apps-api.txt (100%)

diff --git a/Makefile b/Makefile
index 9dadef16..8d1401ff 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ install-pypgstac:
 
 install-lambda-deps:
 	python -m pip install --upgrade pip && \
-	python -m pip install -r apps/api/requirements.txt -t apps/api/src/
+	python -m pip install -r requirements-apps-api.txt -t apps/api/src/
 
 deploy:
 	aws cloudformation package \
diff --git a/apps/api/requirements.txt b/requirements-apps-api.txt
similarity index 100%
rename from apps/api/requirements.txt
rename to requirements-apps-api.txt
diff --git a/requirements.txt b/requirements.txt
index 0e9b1b7b..df9fd64c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
--r apps/api/requirements.txt
+-r requirements-apps-api.txt
 boto3==1.26.3
 cfn-lint==0.71.0
 flake8==5.0.4

From 23ae08d5a79c30f241266b6984f376533cbc4e0e Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Fri, 11 Nov 2022 15:07:20 -0900
Subject: [PATCH 074/144] fix typo in license file name

---
 LICENCE => LICENSE | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename LICENCE => LICENSE (100%)

diff --git a/LICENCE b/LICENSE
similarity index 100%
rename from LICENCE
rename to LICENSE

From 15f70c08fbabae8946ef1355ed08bc6622579e84 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 15:19:59 -0900
Subject: [PATCH 075/144] update deployment workflow

---
 .github/workflows/deploy-stac-api.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 6ec11ea5..6e35e87e 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -3,7 +3,7 @@ name: Deploy STAC API
 on:
   push:
     branches:
-      - sandbox
+      - develop
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -12,12 +12,12 @@ env:
   CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
   DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
   DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
-  STACK_NAME: stac-api-sandbox
+  STACK_NAME: stac-api-test
 
 jobs:
   deploy-stac-api:
     runs-on: ubuntu-latest
-    environment: sandbox
+    environment: test
     steps:
       - uses: actions/checkout@v3
       - uses: aws-actions/configure-aws-credentials@v1

From 19fd5c4f40ab91b7d0c6acbdcb3a3b667429b0cc Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 15:31:28 -0900
Subject: [PATCH 076/144] get github env name from stack name

---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 34fe61cb..d76c2f1b 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -20,7 +20,7 @@ env:
 jobs:
   deploy-stac-api:
     runs-on: ubuntu-latest
-    environment: test
+    environment: ${{ env.STACK_NAME }}
     steps:
       - uses: actions/checkout@v3
       - uses: aws-actions/configure-aws-credentials@v1

From 35e3c91fd4d4784db5b47d95bd490ce266c28dd7 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 11 Nov 2022 15:36:33 -0900
Subject: [PATCH 077/144] rename stack

---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index d76c2f1b..0c31e791 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -6,7 +6,7 @@ on:
       - develop
 
 env:
-  STACK_NAME: stac-api-test
+  STACK_NAME: asf-stac-test
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   CF_TEMPLATES_BUCKET: ${{ secrets.CF_TEMPLATES_BUCKET }}

From 0c648ae199d152cd7145689ae186c32362e6750d Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 10:36:55 -0900
Subject: [PATCH 078/144] update README

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 33e19d65..1b1372e1 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,8 @@ TODO: document creating conda env and installing developer deps
 
 TODO: proofread docs since adding multiple database users
 
+TODO: document database URLs and the `/api.html` endpoint for the Swagger UI
+
 ### Upgrading the database
 
 The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs

From 65e8104d22d4023681f9a03c4e47df7ca31b8de8 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 13:57:52 -0900
Subject: [PATCH 079/144] use matrix in deploy-stac-api.yml

---
 .github/workflows/deploy-stac-api.yml | 51 ++++++++++++---------------
 1 file changed, 23 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 0c31e791..43e8e3ea 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -5,28 +5,23 @@ on:
     branches:
       - develop
 
-env:
-  STACK_NAME: asf-stac-test
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-  CF_TEMPLATES_BUCKET: ${{ secrets.CF_TEMPLATES_BUCKET }}
-  CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
-  DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
-  DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
-  VPC_ID: ${{ secrets.VPC_ID }}
-  SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
-  CIDR_IP: ${{ secrets.CIDR_IP }}
-
 jobs:
   deploy-stac-api:
     runs-on: ubuntu-latest
-    environment: ${{ env.STACK_NAME }}
+    matrix:
+      include:
+        - environment: asf-stac-test
+          template_bucket: cf-templates-118ylv0o6jp2n-us-west-2
+
+    environment:
+      name: ${{ matrix.environment }}
+
     steps:
       - uses: actions/checkout@v3
       - uses: aws-actions/configure-aws-credentials@v1
         with:
-          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
       - uses: actions/setup-python@v3
         with:
@@ -38,28 +33,28 @@ jobs:
       - name: Deploy CloudFormation stack
         run: |
           make -s deploy \
-            s3_bucket=${{ env.CF_TEMPLATES_BUCKET }} \
-            stack_name=${{ env.STACK_NAME }} \
-            cloudformation_role_arn=${{ env.CLOUDFORMATION_ROLE_ARN }} \
-            db_admin_password=${{ env.DB_ADMIN_PASSWORD }} \
-            db_read_password=${{ env.DB_READ_PASSWORD }} \
-            vpc_id=${{ env.VPC_ID }} \
-            subnet_ids=${{ env.SUBNET_IDS }} \
-            cidr_ip=${{ env.CIDR_IP }}
+            stack_name=${{ matrix.environment }} \
+            s3_bucket=${{ matrix.template_bucket }} \
+            cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
+            db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
+            db_read_password=${{ secrets.DB_READ_PASSWORD }} \
+            vpc_id=${{ secrets.VPC_ID }} \
+            subnet_ids=${{ secrets.SUBNET_IDS }} \
+            cidr_ip=${{ secrets.CIDR_IP }}
 
       - name: Get database host
         run: |
           echo "DATABASE_HOST=$(aws cloudformation describe-stacks \
-                                --stack-name $STACK_NAME \
+                                --stack-name ${{ matrix.environment }} \
                                 --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
                                 --output text)" >> $GITHUB_ENV
       - name: Install or upgrade PostGIS
-        run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_admin_password=${{ env.DB_ADMIN_PASSWORD }}
+        run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }}
       - name: Install or upgrade PgSTAC
-        run: make -s pypgstac-migrate db_host=$DATABASE_HOST db_admin_password=${{ env.DB_ADMIN_PASSWORD }}
+        run: make -s pypgstac-migrate db_host=$DATABASE_HOST db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }}
       - name: Configure database roles
         run: |
           make -s configure-database-roles \
             db_host=$DATABASE_HOST \
-            db_admin_password=${{ env.DB_ADMIN_PASSWORD }} \
-            db_read_password=${{ env.DB_READ_PASSWORD }}
+            db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
+            db_read_password=${{ secrets.DB_READ_PASSWORD }}

From 6f9cdac4bf7cd1173c8d9ce83b9754d701aaa11b Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 14:08:24 -0900
Subject: [PATCH 080/144] fix typo in github deploy-stac-api action

---
 .github/workflows/deploy-stac-api.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 43e8e3ea..5ec21a8b 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -8,10 +8,11 @@ on:
 jobs:
   deploy-stac-api:
     runs-on: ubuntu-latest
-    matrix:
-      include:
-        - environment: asf-stac-test
-          template_bucket: cf-templates-118ylv0o6jp2n-us-west-2
+    strategy:
+      matrix:
+        include:
+          - environment: asf-stac-test
+            template_bucket: cf-templates-118ylv0o6jp2n-us-west-2
 
     environment:
       name: ${{ matrix.environment }}

From 8545565b18cd92f5003c12a8df5115808a606378 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 14:11:16 -0900
Subject: [PATCH 081/144] fix template bucket name

---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 5ec21a8b..150d9c51 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -12,7 +12,7 @@ jobs:
       matrix:
         include:
           - environment: asf-stac-test
-            template_bucket: cf-templates-118ylv0o6jp2n-us-west-2
+            template_bucket: cf-templates-118mtzosmrltk-us-west-2
 
     environment:
       name: ${{ matrix.environment }}

From fd42e683b1297ced05e59630a2a0cb31263c6401 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 14:14:45 -0900
Subject: [PATCH 082/144] fix bucket name in deploy workflow (again)

---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 150d9c51..139637d5 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -12,7 +12,7 @@ jobs:
       matrix:
         include:
           - environment: asf-stac-test
-            template_bucket: cf-templates-118mtzosmrltk-us-west-2
+            template_bucket: cf-templates-aubvn3i9olmk-us-west-2
 
     environment:
       name: ${{ matrix.environment }}

From c256442dc464755e9a7cc2465a9c2044554cf77b Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 14:15:36 -0900
Subject: [PATCH 083/144] skip already-existing objects in ingest script

---
 ingest_data.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/ingest_data.py b/ingest_data.py
index eb1eaa1e..b5c02529 100644
--- a/ingest_data.py
+++ b/ingest_data.py
@@ -1,5 +1,6 @@
 """
-Adds a STAC dataset to a STAC API application.
+Adds a STAC dataset to a STAC API application. Skips objects that
+already exist.
 
 Assumes that the dataset is arranged as a tree (connected acyclic
 graph). This means that all objects should be reachable from the root
@@ -43,9 +44,14 @@ def add_stac_object(stac_object: dict, api_url: str) -> None:
     else:
         assert stac_object['type'] == 'Feature'
         endpoint = f'/collections/{stac_object["collection"]}/items'
+
     url = urljoin(api_url, endpoint)
     response = requests.post(url, json=stac_object)
-    response.raise_for_status()
+
+    if response.status_code == 409:
+        print('Skipping existing object')
+    else:
+        response.raise_for_status()
 
 
 def parse_args() -> argparse.Namespace:

From 69ecb085fef602a48ee906383d2895453462ae2a Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 14:24:18 -0900
Subject: [PATCH 084/144] add custom domain name for api

---
 .github/workflows/deploy-stac-api.yml |  5 ++++-
 Makefile                              |  4 +++-
 apps/api/cloudformation.yml           | 21 +++++++++++++++++++++
 apps/cloudformation.yml               |  8 ++++++++
 4 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 139637d5..494c5040 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -13,6 +13,7 @@ jobs:
         include:
           - environment: asf-stac-test
             template_bucket: cf-templates-aubvn3i9olmk-us-west-2
+            domain_name: stac-test.asf.alaska.edu
 
     environment:
       name: ${{ matrix.environment }}
@@ -41,7 +42,9 @@ jobs:
             db_read_password=${{ secrets.DB_READ_PASSWORD }} \
             vpc_id=${{ secrets.VPC_ID }} \
             subnet_ids=${{ secrets.SUBNET_IDS }} \
-            cidr_ip=${{ secrets.CIDR_IP }}
+            cidr_ip=${{ secrets.CIDR_IP }} \
+            domain_name=${{ matrix.domain_name }} \
+            certificate_arn=${{ secrets.CERTIFICATE_ARN }}
 
       - name: Get database host
         run: |
diff --git a/Makefile b/Makefile
index 2ca703ef..320c7424 100644
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,9 @@ deploy:
 	      DatabaseReadPassword=${db_read_password} \
 	      VpcId=${vpc_id} \
 	      SubnetIds=${subnet_ids} \
-	      CidrIp=${cidr_ip}
+	      CidrIp=${cidr_ip} \
+	      DomainName=${domain_name} \
+	      CertificateArn=${certificate_arn}
 
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=${db_user} PGPASSWORD=${db_password} psql
diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 167bf61d..66a7be8c 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -15,6 +15,12 @@ Parameters:
   SecurityGroupId:
     Type: AWS::EC2::SecurityGroup::Id
 
+  DomainName:
+    Type: String
+
+  CertificateArn:
+    Type: String
+
 Resources:
   Lambda:
     Type: AWS::Lambda::Function
@@ -110,3 +116,18 @@ Resources:
               - Effect: Allow
                 Action: lambda:InvokeFunction
                 Resource: !GetAtt Lambda.Arn
+
+  CustomDomainName:
+    Type: AWS::ApiGatewayV2::DomainName
+    Properties:
+      DomainName: !Ref DomainName
+      DomainNameConfigurations:
+        - CertificateArn: !Ref CertificateArn
+          EndpointType: REGIONAL
+
+  ApiMapping:
+    Type: AWS::ApiGatewayV2::ApiMapping
+    Properties:
+      ApiId: !Ref Api
+      DomainName: !Ref CustomDomainName
+      Stage: $default
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 32650a5b..14c3a2b4 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -18,6 +18,12 @@ Parameters:
     Type: String
     NoEcho: true
 
+  DomainName:
+    Type: String
+
+  CertificateArn:
+    Type: String
+
 Outputs:
   DatabaseHost:
     Value: !GetAtt Database.Outputs.DatabaseHost
@@ -31,6 +37,8 @@ Resources:
         DatabaseReadPassword: !Ref DatabaseReadPassword
         SecurityGroupId: !GetAtt Database.Outputs.ClientSecurityGroupId
         SubnetIds: !Join [",", !Ref SubnetIds]
+        DomainName: !Ref DomainName
+        CertificateArn: !Ref CertificateArn
       TemplateURL: api/cloudformation.yml
 
   Database:

From d56bf4ba162070c09b5585d4f9f598052ad068db Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 15:29:45 -0900
Subject: [PATCH 085/144] Add configure-database make target

---
 Makefile | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index 2ca703ef..7cb3fc89 100644
--- a/Makefile
+++ b/Makefile
@@ -30,18 +30,20 @@ deploy:
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=${db_user} PGPASSWORD=${db_password} psql
 
+configure-database: install-or-upgrade-postgis pypgstac-migrate configure-database-roles
+
 install-or-upgrade-postgis:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql \
 	    -f install-or-upgrade-postgis.sql
 
+pypgstac-migrate:
+	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} pypgstac migrate
+
 configure-database-roles:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} psql \
 	    --set=db_read_password=${db_read_password} \
 	    -f configure-database-roles.sql
 
-pypgstac-migrate:
-	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=postgres PGPASSWORD=${db_admin_password} pypgstac migrate
-
 run-api:
 	POSTGRES_HOST_READER=${db_host} POSTGRES_HOST_WRITER=${db_host} POSTGRES_PORT=5432 \
 	    POSTGRES_DBNAME=postgres POSTGRES_USER=postgres POSTGRES_PASS=${db_admin_password} \

From 9a01b6409e161de3827cc9fbf587542b5492429f Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 15:33:32 -0900
Subject: [PATCH 086/144] remove lambda deps from gitignore

---
 .gitignore | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 13a7c943..2c98991e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,2 @@
 # CloudFormation
 packaged.yml
-
-# Lambda dependencies
-apps/api/src/*
-!apps/api/src/api.py

From 082f7c5545bc2bbb5f36b59e9ab630e5a035f5a1 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 15:39:11 -0900
Subject: [PATCH 087/144] add flake8 to static analysis

---
 .github/workflows/static-analysis.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml
index cde098da..38f8ecbc 100644
--- a/.github/workflows/static-analysis.yml
+++ b/.github/workflows/static-analysis.yml
@@ -3,6 +3,16 @@ name: Static code analysis
 on: push
 
 jobs:
+  flake8:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3.1.0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - run: make install
+      - run: make flake8
+
   cfn-lint:
     runs-on: ubuntu-latest
     steps:

From 1a39c04554853a212fecbba6bfe5bab2f867734b Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 15:43:38 -0900
Subject: [PATCH 088/144] delete gitignore

---
 .gitignore | 2 --
 1 file changed, 2 deletions(-)
 delete mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 2c98991e..00000000
--- a/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-# CloudFormation
-packaged.yml

From 0108447228a90cafc5d6faceca01afeda5eea4a4 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 15:46:17 -0900
Subject: [PATCH 089/144] first draft of codebuild project

---
 apps/cloudformation.yml          | 12 ++--
 apps/database/cloudformation.yml | 95 +++++++++++++++++++++++++++++++-
 2 files changed, 97 insertions(+), 10 deletions(-)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 32650a5b..dc4b9952 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -19,8 +19,8 @@ Parameters:
     NoEcho: true
 
 Outputs:
-  DatabaseHost:
-    Value: !GetAtt Database.Outputs.DatabaseHost
+  BuildProject:
+    Value: !GetAtt Database.Outputs.BuildProject
 
 Resources:
   Api:
@@ -38,12 +38,8 @@ Resources:
     Properties:
       Parameters:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
+        DatabaseReadPassword: !Ref DatabaseReadPassword
         VpcId: !Ref VpcId
+        SubnetIds: !Join [",", !Ref SubnetIds]
         CidrIp: !Ref CidrIp
       TemplateURL: database/cloudformation.yml
-
-  DatabaseSecret:
-    Type: AWS::SecretsManager::Secret
-    Properties:
-      Description: !Sub "${AWS::StackName} database credentials"
-      SecretString: !Sub '{"database_host": "${Database.Outputs.DatabaseHost}", "admin_user": "postgres", "admin_password": "${DatabaseAdminPassword}", "read_user": "pgstac_read", "read_password": "${DatabaseReadPassword}"}'
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 4421f5d4..cabc482a 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -1,19 +1,27 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
+
   DatabaseAdminPassword:
     Type: String
     NoEcho: true
 
+  DatabaseReadPassword:
+    Type: String
+    NoEcho: true
+
   VpcId:
     Type: AWS::EC2::VPC::Id
 
+  SubnetIds:
+    Type: CommaDelimitedList
+
   CidrIp:
     Type: String
 
 Outputs:
-  DatabaseHost:
-    Value: !GetAtt DatabaseInstance.Endpoint.Address
+  BuildProject:
+    Value: !Ref CodeBuildProject
 
   ClientSecurityGroupId:
     Value: !Ref ClientSecurityGroup
@@ -49,3 +57,86 @@ Resources:
     Properties:
       GroupDescription: !Sub "Security group for clients of ${AWS::StackName} database"
       VpcId: !Ref VpcId
+
+  DatabaseSecret:
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: !Sub "${AWS::StackName} database credentials"
+      SecretString: !Sub '{"database_host": "${DatabaseInstance.Endpoint.Address}", "admin_user": "postgres", "admin_password": "${DatabaseAdminPassword}", "read_user": "pgstac_read", "read_password": "${DatabaseReadPassword}"}'
+
+  CodeBuildProject:
+    Type: AWS::CodeBuild::Project
+    Properties:
+      Environment:
+        ComputeType: BUILD_GENERAL1_SMALL
+        Type: LINUX_CONTAINER
+        Image: 	aws/codebuild/standard:6.0
+        EnvironmentVariables:
+          - Name: PGHOST
+            Type: SECRETS_MANAGER
+            Value: !Sub "${DatabaseSecret}:database_host"
+          - Name: PGUSER
+            Type: SECRETS_MANAGER
+            Value: !Sub "${DatabaseSecret}:admin_user"
+          - Name: PGPASSWORD
+            Type: SECRETS_MANAGER
+            Value: !Sub "${DatabaseSecret}:admin_password"
+#      LogsConfig:
+      ServiceRole: !Ref CodeBuildServiceRole
+      Source:
+        Type: GITHUB
+        Location: https://github.com/ASFHyP3/asf-stac.git
+        BuildSpec: make cfn-lint
+      SourceVersion: develop
+      Artifacts:
+        Type: NO_ARTIFACTS
+      VpcConfig:
+        VpcId: !Ref VpcId
+        Subnets: !Ref SubnetIds
+        SecurityGroupIds:
+          - !Ref ClientSecurityGroup
+
+  CodeBuildServiceRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: 2012-10-17
+        Statement:
+          Action: sts:AssumeRole
+          Principal:
+            Service: codebuild.amazonaws.com
+          Effect: Allow
+      Policies:
+        - PolicyName: policy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Sid: CloudWatchLogsPolicy
+                Effect: Allow
+                Action:
+                - logs:CreateLogGroup
+                - logs:CreateLogStream
+                - logs:PutLogEvents
+                Resource: "*"
+              - Sid: CodeCommitPolicy
+                Effect: Allow
+                Action:
+                - codecommit:GitPull
+                Resource: "*"
+              - Sid: S3GetObjectPolicy
+                Effect: Allow
+                Action:
+                - s3:GetObject
+                - s3:GetObjectVersion
+                Resource: "*"
+              - Sid: S3PutObjectPolicy
+                Effect: Allow
+                Action:
+                - s3:PutObject
+                Resource: "*"
+              - Sid: S3BucketIdentity
+                Effect: Allow
+                Action:
+                - s3:GetBucketAcl
+                - s3:GetBucketLocation
+                Resource: "*"

From 35b6afeb8ff0c2b06506f360d62bfd1e04721933 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Mon, 14 Nov 2022 15:58:26 -0900
Subject: [PATCH 090/144] Run CodeBuild from deploy workflow

---
 .github/workflows/deploy-stac-api.yml | 23 +++++++----------------
 1 file changed, 7 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 139637d5..2cd462fd 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -42,20 +42,11 @@ jobs:
             vpc_id=${{ secrets.VPC_ID }} \
             subnet_ids=${{ secrets.SUBNET_IDS }} \
             cidr_ip=${{ secrets.CIDR_IP }}
-
-      - name: Get database host
-        run: |
-          echo "DATABASE_HOST=$(aws cloudformation describe-stacks \
-                                --stack-name ${{ matrix.environment }} \
-                                --query 'Stacks[0].Outputs[?OutputKey==`DatabaseHost`].OutputValue' \
-                                --output text)" >> $GITHUB_ENV
-      - name: Install or upgrade PostGIS
-        run: make -s install-or-upgrade-postgis db_host=$DATABASE_HOST db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }}
-      - name: Install or upgrade PgSTAC
-        run: make -s pypgstac-migrate db_host=$DATABASE_HOST db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }}
-      - name: Configure database roles
+      - name: Get CodeBuild project
         run: |
-          make -s configure-database-roles \
-            db_host=$DATABASE_HOST \
-            db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
-            db_read_password=${{ secrets.DB_READ_PASSWORD }}
+          echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
+                                    --stack-name ${{ matrix.environment }} \
+                                    --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
+                                    --output text)" >> $GITHUB_ENV
+      - name: Run CodeBuild
+        run: aws codebuild start-build --project-name $CODEBUILD_PROJECT

From ad4ba49dea5f960fd7c853233e055b5d1ed19893 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Mon, 14 Nov 2022 16:26:19 -0900
Subject: [PATCH 091/144] clean up codebuild cloudformation

---
 apps/database/cloudformation.yml | 55 ++++++++++++++++----------------
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index cabc482a..0cb9665a 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -70,7 +70,7 @@ Resources:
       Environment:
         ComputeType: BUILD_GENERAL1_SMALL
         Type: LINUX_CONTAINER
-        Image: 	aws/codebuild/standard:6.0
+        Image: aws/codebuild/standard:6.0
         EnvironmentVariables:
           - Name: PGHOST
             Type: SECRETS_MANAGER
@@ -86,6 +86,7 @@ Resources:
       Source:
         Type: GITHUB
         Location: https://github.com/ASFHyP3/asf-stac.git
+        GitCloneDepth: 1
         BuildSpec: make cfn-lint
       SourceVersion: develop
       Artifacts:
@@ -111,32 +112,30 @@ Resources:
           PolicyDocument:
             Version: '2012-10-17'
             Statement:
-              - Sid: CloudWatchLogsPolicy
-                Effect: Allow
+              - Effect: Allow
                 Action:
-                - logs:CreateLogGroup
-                - logs:CreateLogStream
-                - logs:PutLogEvents
-                Resource: "*"
-              - Sid: CodeCommitPolicy
-                Effect: Allow
-                Action:
-                - codecommit:GitPull
-                Resource: "*"
-              - Sid: S3GetObjectPolicy
-                Effect: Allow
-                Action:
-                - s3:GetObject
-                - s3:GetObjectVersion
-                Resource: "*"
-              - Sid: S3PutObjectPolicy
-                Effect: Allow
-                Action:
-                - s3:PutObject
-                Resource: "*"
-              - Sid: S3BucketIdentity
-                Effect: Allow
-                Action:
-                - s3:GetBucketAcl
-                - s3:GetBucketLocation
+                  - logs:CreateLogGroup
+                  - logs:CreateLogStream
+                  - logs:PutLogEvents
+                  - codecommit:GitPull
+                  - s3:GetObject
+                  - s3:GetObjectVersion
+                  - s3:PutObject
+                  - s3:GetBucketAcl
+                  - s3:GetBucketLocation
+                  - ec2:CreateNetworkInterface
+                  - ec2:DescribeDhcpOptions
+                  - ec2:DescribeNetworkInterfaces
+                  - ec2:DeleteNetworkInterface
+                  - ec2:DescribeSubnets
+                  - ec2:DescribeSecurityGroups
+                  - ec2:DescribeVpcs
                 Resource: "*"
+              - Effect: Allow
+                Action: ec2:CreateNetworkInterfacePermission
+                Resource: !Sub "arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"
+                Condition:
+                  StringEquals:
+                    ec2:AuthorizedService: codebuild.amazonaws.com
+                  StringLike:
+                    ec2:Subnet: !Sub "arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:subnet/*"

From 27c4f939b5b3d0bc0f02486e174f44177edb6be5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Nov 2022 11:10:05 +0000
Subject: [PATCH 092/144] Bump actions/setup-python from 3 to 4

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 3 to 4.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](https://github.com/actions/setup-python/compare/v3...v4)

---
updated-dependencies:
- dependency-name: actions/setup-python
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 494c5040..e09566b2 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -25,7 +25,7 @@ jobs:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: 3.9
       - name: Install Lambda dependencies

From e030afceb31c2b5f09df19a8dbf638e2de48d1b5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Nov 2022 11:10:17 +0000
Subject: [PATCH 093/144] Bump cfn-lint from 0.71.0 to 0.71.1

Bumps [cfn-lint](https://github.com/aws-cloudformation/cfn-python-lint) from 0.71.0 to 0.71.1.
- [Release notes](https://github.com/aws-cloudformation/cfn-python-lint/releases)
- [Changelog](https://github.com/aws-cloudformation/cfn-lint/blob/main/CHANGELOG.md)
- [Commits](https://github.com/aws-cloudformation/cfn-python-lint/compare/v0.71.0...v0.71.1)

---
updated-dependencies:
- dependency-name: cfn-lint
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index df9fd64c..fac856fc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 -r requirements-apps-api.txt
 boto3==1.26.3
-cfn-lint==0.71.0
+cfn-lint==0.71.1
 flake8==5.0.4
 pypgstac[psycopg]==0.6.10
 pystac==1.6.1

From d94358ecacb8c4e1f476e42b845e5b3a3e81180d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Nov 2022 17:58:04 +0000
Subject: [PATCH 094/144] Bump boto3 from 1.26.3 to 1.26.9

Bumps [boto3](https://github.com/boto/boto3) from 1.26.3 to 1.26.9.
- [Release notes](https://github.com/boto/boto3/releases)
- [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst)
- [Commits](https://github.com/boto/boto3/compare/1.26.3...1.26.9)

---
updated-dependencies:
- dependency-name: boto3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index fac856fc..4a49eb73 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 -r requirements-apps-api.txt
-boto3==1.26.3
+boto3==1.26.9
 cfn-lint==0.71.1
 flake8==5.0.4
 pypgstac[psycopg]==0.6.10

From 3c3c1586973ca50bfeaa55cf1b393db8e3081e64 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Tue, 15 Nov 2022 15:35:10 -0900
Subject: [PATCH 095/144] create vpc as part of cloudformation stack

---
 apps/api/cloudformation.yml      |  8 +--
 apps/cloudformation.yml          | 18 ++++---
 apps/database/cloudformation.yml | 10 ++--
 apps/vpc/cloudformation.yml      | 93 ++++++++++++++++++++++++++++++++
 4 files changed, 114 insertions(+), 15 deletions(-)
 create mode 100644 apps/vpc/cloudformation.yml

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index 167bf61d..cd224732 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -1,6 +1,7 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
+
   DatabaseHost:
     Type: String
     NoEcho: true
@@ -9,8 +10,8 @@ Parameters:
     Type: String
     NoEcho: true
 
-  SubnetIds:
-    Type: CommaDelimitedList
+  SubnetId:
+    Type: AWS::EC2::Subnet::Id
 
   SecurityGroupId:
     Type: AWS::EC2::SecurityGroup::Id
@@ -36,7 +37,8 @@ Resources:
       VpcConfig:
         SecurityGroupIds:
           - !Ref SecurityGroupId
-        SubnetIds: !Ref SubnetIds
+        SubnetIds:
+          - !Ref SubnetId
 
   LambdaLogGroup:
     Type: AWS::Logs::LogGroup
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index dc4b9952..963a4925 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -1,11 +1,6 @@
 AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
-  VpcId:
-    Type: AWS::EC2::VPC::Id
-
-  SubnetIds:
-    Type: List<AWS::EC2::Subnet::Id>
 
   CidrIp:
     Type: String
@@ -19,10 +14,17 @@ Parameters:
     NoEcho: true
 
 Outputs:
+
   BuildProject:
     Value: !GetAtt Database.Outputs.BuildProject
 
 Resources:
+
+  VPC:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      TemplateURL: vpc/cloudformation.yml
+
   Api:
     Type: AWS::CloudFormation::Stack
     Properties:
@@ -30,7 +32,7 @@ Resources:
         DatabaseHost: !GetAtt Database.Outputs.DatabaseHost
         DatabaseReadPassword: !Ref DatabaseReadPassword
         SecurityGroupId: !GetAtt Database.Outputs.ClientSecurityGroupId
-        SubnetIds: !Join [",", !Ref SubnetIds]
+        SubnetId: !GetAtt VPC.Outputs.PrivateSubnetId
       TemplateURL: api/cloudformation.yml
 
   Database:
@@ -39,7 +41,7 @@ Resources:
       Parameters:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
         DatabaseReadPassword: !Ref DatabaseReadPassword
-        VpcId: !Ref VpcId
-        SubnetIds: !Join [",", !Ref SubnetIds]
+        VpcId: !GetAtt VPC.Outputs.VpcId
+        SubnetId: !GetAtt VPC.Outputs.PrivateSubnetId
         CidrIp: !Ref CidrIp
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 0cb9665a..f35aef5d 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -13,8 +13,8 @@ Parameters:
   VpcId:
     Type: AWS::EC2::VPC::Id
 
-  SubnetIds:
-    Type: CommaDelimitedList
+  SubnetId:
+    Type: AWS::EC2::Subnet::Id
 
   CidrIp:
     Type: String
@@ -27,11 +27,12 @@ Outputs:
     Value: !Ref ClientSecurityGroup
 
 Resources:
+
   DatabaseInstance:
     Type: AWS::RDS::DBInstance
     Properties:
       AllocatedStorage: '20' # TODO depends on StorageType?
-      DBInstanceClass: 'db.t3.micro' # TODO different class for prod?
+      DBInstanceClass: db.t3.micro # TODO different class for prod?
       DBSecurityGroups:
         - !Ref DatabaseSecurityGroup
       Engine: postgres
@@ -93,7 +94,8 @@ Resources:
         Type: NO_ARTIFACTS
       VpcConfig:
         VpcId: !Ref VpcId
-        Subnets: !Ref SubnetIds
+        Subnets:
+          - !Ref SubnetId
         SecurityGroupIds:
           - !Ref ClientSecurityGroup
 
diff --git a/apps/vpc/cloudformation.yml b/apps/vpc/cloudformation.yml
new file mode 100644
index 00000000..a8ecaccd
--- /dev/null
+++ b/apps/vpc/cloudformation.yml
@@ -0,0 +1,93 @@
+Outputs:
+
+  VpcId:
+    Value: !Ref VPC
+
+  PublicSubnetId:
+    Value: !Ref PublicSubnet
+
+  PrivateSubnetId:
+    Value: !Ref PrivateSubnet
+
+Resources:
+
+  VPC:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: 10.192.0.0/16
+      EnableDnsSupport: true
+      EnableDnsHostnames: true
+
+  InternetGateway:
+    Type: AWS::EC2::InternetGateway
+
+  InternetGatewayAttachment:
+    Type: AWS::EC2::VPCGatewayAttachment
+    Properties:
+      InternetGatewayId: !Ref InternetGateway
+      VpcId: !Ref VPC
+
+  PublicSubnet:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [0, !GetAZs '']
+      CidrBlock: 10.192.10.0/24
+      MapPublicIpOnLaunch: true
+
+  PrivateSubnet:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [0, !GetAZs '']
+      CidrBlock: 10.192.20.0/24
+      MapPublicIpOnLaunch: false
+
+  NatGatewayEIP:
+    Type: AWS::EC2::EIP
+    DependsOn: InternetGatewayAttachment
+    Properties:
+      Domain: vpc
+
+  NatGateway:
+    Type: AWS::EC2::NatGateway
+    Properties:
+      AllocationId: !GetAtt NatGatewayEIP.AllocationId
+      SubnetId: !Ref PublicSubnet
+
+  PublicRouteTable:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref VPC
+
+  DefaultPublicRoute:
+    Type: AWS::EC2::Route
+    DependsOn: InternetGatewayAttachment
+    Properties:
+      RouteTableId: !Ref PublicRouteTable
+      DestinationCidrBlock: 0.0.0.0/0
+      GatewayId: !Ref InternetGateway
+
+  PublicSubnetRouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PublicRouteTable
+      SubnetId: !Ref PublicSubnet
+
+  PrivateRouteTable:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref VPC
+
+  DefaultPrivateRoute:
+    Type: AWS::EC2::Route
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable
+      DestinationCidrBlock: 0.0.0.0/0
+      NatGatewayId: !Ref NatGateway
+
+  PrivateSubnetRouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable
+      SubnetId: !Ref PrivateSubnet

From 222c8bb136364201e8b9eecf18ded6c7b68d1bcc Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Tue, 15 Nov 2022 17:54:41 -0900
Subject: [PATCH 096/144] more cloudformation updates

---
 apps/api/cloudformation.yml      |   7 +-
 apps/cloudformation.yml          |   8 +-
 apps/database/buildspec.yml      |  12 ++
 apps/database/cloudformation.yml |  41 +++++--
 apps/vpc/cloudformation.yml      | 195 ++++++++++++++++++++++++++-----
 5 files changed, 213 insertions(+), 50 deletions(-)
 create mode 100644 apps/database/buildspec.yml

diff --git a/apps/api/cloudformation.yml b/apps/api/cloudformation.yml
index cd224732..e1d2d351 100644
--- a/apps/api/cloudformation.yml
+++ b/apps/api/cloudformation.yml
@@ -10,8 +10,8 @@ Parameters:
     Type: String
     NoEcho: true
 
-  SubnetId:
-    Type: AWS::EC2::Subnet::Id
+  SubnetIds:
+    Type: CommaDelimitedList
 
   SecurityGroupId:
     Type: AWS::EC2::SecurityGroup::Id
@@ -37,8 +37,7 @@ Resources:
       VpcConfig:
         SecurityGroupIds:
           - !Ref SecurityGroupId
-        SubnetIds:
-          - !Ref SubnetId
+        SubnetIds: !Ref SubnetIds
 
   LambdaLogGroup:
     Type: AWS::Logs::LogGroup
diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 963a4925..02dc9917 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -23,6 +23,8 @@ Resources:
   VPC:
     Type: AWS::CloudFormation::Stack
     Properties:
+      Parameters:
+        EnvironmentName: !Ref AWS::StackName
       TemplateURL: vpc/cloudformation.yml
 
   Api:
@@ -32,7 +34,7 @@ Resources:
         DatabaseHost: !GetAtt Database.Outputs.DatabaseHost
         DatabaseReadPassword: !Ref DatabaseReadPassword
         SecurityGroupId: !GetAtt Database.Outputs.ClientSecurityGroupId
-        SubnetId: !GetAtt VPC.Outputs.PrivateSubnetId
+        SubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
       TemplateURL: api/cloudformation.yml
 
   Database:
@@ -41,7 +43,7 @@ Resources:
       Parameters:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
         DatabaseReadPassword: !Ref DatabaseReadPassword
-        VpcId: !GetAtt VPC.Outputs.VpcId
-        SubnetId: !GetAtt VPC.Outputs.PrivateSubnetId
+        VpcId: !GetAtt VPC.Outputs.VPC
+        SubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
         CidrIp: !Ref CidrIp
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
new file mode 100644
index 00000000..afb4e789
--- /dev/null
+++ b/apps/database/buildspec.yml
@@ -0,0 +1,12 @@
+version: 0.2
+
+phases:
+  install:
+    runtime-versions:
+      python: 3.9
+    commands:
+      - apt update
+      - apt install postgresql-12
+  build:
+    commands:
+      - make configure-database
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index f35aef5d..0b15f2de 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -13,19 +13,23 @@ Parameters:
   VpcId:
     Type: AWS::EC2::VPC::Id
 
-  SubnetId:
-    Type: AWS::EC2::Subnet::Id
+  SubnetIds:
+    Type: CommaDelimitedList
 
   CidrIp:
     Type: String
 
 Outputs:
+
   BuildProject:
     Value: !Ref CodeBuildProject
 
   ClientSecurityGroupId:
     Value: !Ref ClientSecurityGroup
 
+  DatabaseHost:
+    Value: !GetAtt DatabaseInstance.Endpoint.Address
+
 Resources:
 
   DatabaseInstance:
@@ -33,7 +37,8 @@ Resources:
     Properties:
       AllocatedStorage: '20' # TODO depends on StorageType?
       DBInstanceClass: db.t3.micro # TODO different class for prod?
-      DBSecurityGroups:
+      DBSubnetGroupName: !Ref DatabaseSubnetGroup
+      VPCSecurityGroups:
         - !Ref DatabaseSecurityGroup
       Engine: postgres
       EngineVersion: '14.4'
@@ -45,13 +50,26 @@ Resources:
       PubliclyAccessible: true
       # TODO: StorageType: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbinstance.html#cfn-rds-dbinstance-storagetype
 
+  DatabaseSubnetGroup:
+    Type: AWS::RDS::DBSubnetGroup
+    Properties:
+      DBSubnetGroupDescription: !Sub "Subnet Group for ${AWS::StackName} database"
+      SubnetIds: !Ref SubnetIds
+
   DatabaseSecurityGroup:
-    Type: AWS::RDS::DBSecurityGroup
+    Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupDescription: !Sub "Security group for ${AWS::StackName} database"
-      DBSecurityGroupIngress:
-        - EC2SecurityGroupId: !Ref ClientSecurityGroup
-        - CIDRIP: !Ref CidrIp
+      GroupDescription: !Sub "Security group for clients of ${AWS::StackName} database"
+      VpcId: !Ref VpcId
+      SecurityGroupIngress:
+        - IpProtocol: tcp
+          CidrIp: !Ref CidrIp
+          FromPort: 5432
+          ToPort: 5432
+        - IpProtocol: tcp
+          SourceSecurityGroupId: !Ref ClientSecurityGroup
+          FromPort: 5432
+          ToPort: 5432
 
   ClientSecurityGroup:
     Type: AWS::EC2::SecurityGroup
@@ -71,7 +89,7 @@ Resources:
       Environment:
         ComputeType: BUILD_GENERAL1_SMALL
         Type: LINUX_CONTAINER
-        Image: aws/codebuild/standard:6.0
+        Image: aws/codebuild/standard:5.0
         EnvironmentVariables:
           - Name: PGHOST
             Type: SECRETS_MANAGER
@@ -88,14 +106,13 @@ Resources:
         Type: GITHUB
         Location: https://github.com/ASFHyP3/asf-stac.git
         GitCloneDepth: 1
-        BuildSpec: make cfn-lint
+        BuildSpec: apps/database/buildspec.yml
       SourceVersion: develop
       Artifacts:
         Type: NO_ARTIFACTS
       VpcConfig:
         VpcId: !Ref VpcId
-        Subnets:
-          - !Ref SubnetId
+        Subnets: !Ref SubnetIds
         SecurityGroupIds:
           - !Ref ClientSecurityGroup
 
diff --git a/apps/vpc/cloudformation.yml b/apps/vpc/cloudformation.yml
index a8ecaccd..28193f9f 100644
--- a/apps/vpc/cloudformation.yml
+++ b/apps/vpc/cloudformation.yml
@@ -1,25 +1,55 @@
-Outputs:
-
-  VpcId:
-    Value: !Ref VPC
-
-  PublicSubnetId:
-    Value: !Ref PublicSubnet
-
-  PrivateSubnetId:
-    Value: !Ref PrivateSubnet
+Description:  This template deploys a VPC, with a pair of public and private subnets spread
+  across two Availability Zones. It deploys an internet gateway, with a default
+  route on the public subnets. It deploys a pair of NAT gateways (one in each AZ),
+  and default routes for them in the private subnets.
+
+Parameters:
+  EnvironmentName:
+    Description: An environment name that is prefixed to resource names
+    Type: String
+
+  VpcCIDR:
+    Description: Please enter the IP range (CIDR notation) for this VPC
+    Type: String
+    Default: 10.192.0.0/16
+
+  PublicSubnet1CIDR:
+    Description: Please enter the IP range (CIDR notation) for the public subnet in the first Availability Zone
+    Type: String
+    Default: 10.192.10.0/24
+
+  PublicSubnet2CIDR:
+    Description: Please enter the IP range (CIDR notation) for the public subnet in the second Availability Zone
+    Type: String
+    Default: 10.192.11.0/24
+
+  PrivateSubnet1CIDR:
+    Description: Please enter the IP range (CIDR notation) for the private subnet in the first Availability Zone
+    Type: String
+    Default: 10.192.20.0/24
+
+  PrivateSubnet2CIDR:
+    Description: Please enter the IP range (CIDR notation) for the private subnet in the second Availability Zone
+    Type: String
+    Default: 10.192.21.0/24
 
 Resources:
-
   VPC:
     Type: AWS::EC2::VPC
     Properties:
-      CidrBlock: 10.192.0.0/16
+      CidrBlock: !Ref VpcCIDR
       EnableDnsSupport: true
       EnableDnsHostnames: true
+      Tags:
+        - Key: Name
+          Value: !Ref EnvironmentName
 
   InternetGateway:
     Type: AWS::EC2::InternetGateway
+    Properties:
+      Tags:
+        - Key: Name
+          Value: !Ref EnvironmentName
 
   InternetGatewayAttachment:
     Type: AWS::EC2::VPCGatewayAttachment
@@ -27,38 +57,81 @@ Resources:
       InternetGatewayId: !Ref InternetGateway
       VpcId: !Ref VPC
 
-  PublicSubnet:
+  PublicSubnet1:
     Type: AWS::EC2::Subnet
     Properties:
       VpcId: !Ref VPC
-      AvailabilityZone: !Select [0, !GetAZs '']
-      CidrBlock: 10.192.10.0/24
+      AvailabilityZone: !Select [ 0, !GetAZs '' ]
+      CidrBlock: !Ref PublicSubnet1CIDR
       MapPublicIpOnLaunch: true
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Public Subnet (AZ1)
+
+  PublicSubnet2:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
+      CidrBlock: !Ref PublicSubnet2CIDR
+      MapPublicIpOnLaunch: true
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Public Subnet (AZ2)
+
+  PrivateSubnet1:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [ 0, !GetAZs  '' ]
+      CidrBlock: !Ref PrivateSubnet1CIDR
+      MapPublicIpOnLaunch: false
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Subnet (AZ1)
 
-  PrivateSubnet:
+  PrivateSubnet2:
     Type: AWS::EC2::Subnet
     Properties:
       VpcId: !Ref VPC
-      AvailabilityZone: !Select [0, !GetAZs '']
-      CidrBlock: 10.192.20.0/24
+      AvailabilityZone: !Select [ 1, !GetAZs  '' ]
+      CidrBlock: !Ref PrivateSubnet2CIDR
       MapPublicIpOnLaunch: false
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Subnet (AZ2)
 
-  NatGatewayEIP:
+  NatGateway1EIP:
     Type: AWS::EC2::EIP
     DependsOn: InternetGatewayAttachment
     Properties:
       Domain: vpc
 
-  NatGateway:
+  NatGateway2EIP:
+    Type: AWS::EC2::EIP
+    DependsOn: InternetGatewayAttachment
+    Properties:
+      Domain: vpc
+
+  NatGateway1:
+    Type: AWS::EC2::NatGateway
+    Properties:
+      AllocationId: !GetAtt NatGateway1EIP.AllocationId
+      SubnetId: !Ref PublicSubnet1
+
+  NatGateway2:
     Type: AWS::EC2::NatGateway
     Properties:
-      AllocationId: !GetAtt NatGatewayEIP.AllocationId
-      SubnetId: !Ref PublicSubnet
+      AllocationId: !GetAtt NatGateway2EIP.AllocationId
+      SubnetId: !Ref PublicSubnet2
 
   PublicRouteTable:
     Type: AWS::EC2::RouteTable
     Properties:
       VpcId: !Ref VPC
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Public Routes
 
   DefaultPublicRoute:
     Type: AWS::EC2::Route
@@ -68,26 +141,86 @@ Resources:
       DestinationCidrBlock: 0.0.0.0/0
       GatewayId: !Ref InternetGateway
 
-  PublicSubnetRouteTableAssociation:
+  PublicSubnet1RouteTableAssociation:
     Type: AWS::EC2::SubnetRouteTableAssociation
     Properties:
       RouteTableId: !Ref PublicRouteTable
-      SubnetId: !Ref PublicSubnet
+      SubnetId: !Ref PublicSubnet1
 
-  PrivateRouteTable:
+  PublicSubnet2RouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PublicRouteTable
+      SubnetId: !Ref PublicSubnet2
+
+
+  PrivateRouteTable1:
     Type: AWS::EC2::RouteTable
     Properties:
       VpcId: !Ref VPC
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Routes (AZ1)
 
-  DefaultPrivateRoute:
+  DefaultPrivateRoute1:
     Type: AWS::EC2::Route
     Properties:
-      RouteTableId: !Ref PrivateRouteTable
+      RouteTableId: !Ref PrivateRouteTable1
       DestinationCidrBlock: 0.0.0.0/0
-      NatGatewayId: !Ref NatGateway
+      NatGatewayId: !Ref NatGateway1
 
-  PrivateSubnetRouteTableAssociation:
+  PrivateSubnet1RouteTableAssociation:
     Type: AWS::EC2::SubnetRouteTableAssociation
     Properties:
-      RouteTableId: !Ref PrivateRouteTable
-      SubnetId: !Ref PrivateSubnet
+      RouteTableId: !Ref PrivateRouteTable1
+      SubnetId: !Ref PrivateSubnet1
+
+  PrivateRouteTable2:
+    Type: AWS::EC2::RouteTable
+    Properties:
+      VpcId: !Ref VPC
+      Tags:
+        - Key: Name
+          Value: !Sub ${EnvironmentName} Private Routes (AZ2)
+
+  DefaultPrivateRoute2:
+    Type: AWS::EC2::Route
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable2
+      DestinationCidrBlock: 0.0.0.0/0
+      NatGatewayId: !Ref NatGateway2
+
+  PrivateSubnet2RouteTableAssociation:
+    Type: AWS::EC2::SubnetRouteTableAssociation
+    Properties:
+      RouteTableId: !Ref PrivateRouteTable2
+      SubnetId: !Ref PrivateSubnet2
+
+Outputs:
+  VPC:
+    Description: A reference to the created VPC
+    Value: !Ref VPC
+
+  PublicSubnets:
+    Description: A list of the public subnets
+    Value: !Join [ ",", [ !Ref PublicSubnet1, !Ref PublicSubnet2 ]]
+
+  PrivateSubnets:
+    Description: A list of the private subnets
+    Value: !Join [ ",", [ !Ref PrivateSubnet1, !Ref PrivateSubnet2 ]]
+
+  PublicSubnet1:
+    Description: A reference to the public subnet in the 1st Availability Zone
+    Value: !Ref PublicSubnet1
+
+  PublicSubnet2:
+    Description: A reference to the public subnet in the 2nd Availability Zone
+    Value: !Ref PublicSubnet2
+
+  PrivateSubnet1:
+    Description: A reference to the private subnet in the 1st Availability Zone
+    Value: !Ref PrivateSubnet1
+
+  PrivateSubnet2:
+    Description: A reference to the private subnet in the 2nd Availability Zone
+    Value: !Ref PrivateSubnet2

From ac6e673df0503a304ea4594d5a7fb67a0e7e8f01 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Tue, 15 Nov 2022 18:04:12 -0900
Subject: [PATCH 097/144] github branch parameter, secretsmanager permissions
 for codebuild, buildspec typo fix

---
 apps/cloudformation.yml          | 4 ++++
 apps/database/buildspec.yml      | 2 +-
 apps/database/cloudformation.yml | 8 +++++++-
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 02dc9917..31687952 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -2,6 +2,9 @@ AWSTemplateFormatVersion: 2010-09-09
 
 Parameters:
 
+  GithubBranch:
+    Type: String
+
   CidrIp:
     Type: String
 
@@ -46,4 +49,5 @@ Resources:
         VpcId: !GetAtt VPC.Outputs.VPC
         SubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
         CidrIp: !Ref CidrIp
+        GithubBranch: !Ref GithubBranch
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index afb4e789..d7223505 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -6,7 +6,7 @@ phases:
       python: 3.9
     commands:
       - apt update
-      - apt install postgresql-12
+      - apt install -y postgresql-12
   build:
     commands:
       - make configure-database
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 0b15f2de..cf4888fb 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -19,6 +19,9 @@ Parameters:
   CidrIp:
     Type: String
 
+  GithubBranch:
+    Type: String
+
 Outputs:
 
   BuildProject:
@@ -107,7 +110,7 @@ Resources:
         Location: https://github.com/ASFHyP3/asf-stac.git
         GitCloneDepth: 1
         BuildSpec: apps/database/buildspec.yml
-      SourceVersion: develop
+      SourceVersion: !Ref GithubBranch
       Artifacts:
         Type: NO_ARTIFACTS
       VpcConfig:
@@ -150,6 +153,9 @@ Resources:
                   - ec2:DescribeSecurityGroups
                   - ec2:DescribeVpcs
                 Resource: "*"
+              - Effect: Allow
+                Action: secretsmanager:GetSecretValue
+                Resource: !Ref DatabaseSecret
               - Effect: Allow
                 Action: ec2:CreateNetworkInterfacePermission
                 Resource: !Sub "arn:aws:ec2:${AWS::Region}:${AWS::AccountId}:network-interface/*"

From 346abc2cb822a04b3f2ded2b3aa4471fae3f83a6 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Tue, 15 Nov 2022 18:14:56 -0900
Subject: [PATCH 098/144] pass environment variables to make in buildspec

---
 apps/database/buildspec.yml      | 2 +-
 apps/database/cloudformation.yml | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index d7223505..215b3779 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -9,4 +9,4 @@ phases:
       - apt install -y postgresql-12
   build:
     commands:
-      - make configure-database
+      - make configure-database db_host=$DBHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index cf4888fb..502582dd 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -103,6 +103,9 @@ Resources:
           - Name: PGPASSWORD
             Type: SECRETS_MANAGER
             Value: !Sub "${DatabaseSecret}:admin_password"
+          - Name: READ_PASSWORD
+            Type: SECRETS_MANAGER
+            Value: !Sub "${DatabaseSecret}:read_password"
 #      LogsConfig:
       ServiceRole: !Ref CodeBuildServiceRole
       Source:

From f61024e29ec36981e4c48763f5329d773fb2e64c Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Tue, 15 Nov 2022 18:19:18 -0900
Subject: [PATCH 099/144] update codebuild environment variables

---
 apps/database/buildspec.yml      | 2 +-
 apps/database/cloudformation.yml | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index 215b3779..779ef8c2 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -9,4 +9,4 @@ phases:
       - apt install -y postgresql-12
   build:
     commands:
-      - make configure-database db_host=$DBHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD
+      - make configure-database db_host=$PGHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 502582dd..0ad94976 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -97,9 +97,6 @@ Resources:
           - Name: PGHOST
             Type: SECRETS_MANAGER
             Value: !Sub "${DatabaseSecret}:database_host"
-          - Name: PGUSER
-            Type: SECRETS_MANAGER
-            Value: !Sub "${DatabaseSecret}:admin_user"
           - Name: PGPASSWORD
             Type: SECRETS_MANAGER
             Value: !Sub "${DatabaseSecret}:admin_password"

From 2f05cb687db058aecafbfe7f2921fd99f308c04e Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Tue, 15 Nov 2022 18:24:01 -0900
Subject: [PATCH 100/144] install pypgstac in buildspec

---
 apps/database/buildspec.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index 779ef8c2..b4194708 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -7,6 +7,7 @@ phases:
     commands:
       - apt update
       - apt install -y postgresql-12
+      - make install-pypgstac
   build:
     commands:
       - make configure-database db_host=$PGHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD

From c31b8a0f2f107ef908828d96d23c363f41d6aa26 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 16 Nov 2022 11:28:15 +0000
Subject: [PATCH 101/144] Bump boto3 from 1.26.9 to 1.26.10

Bumps [boto3](https://github.com/boto/boto3) from 1.26.9 to 1.26.10.
- [Release notes](https://github.com/boto/boto3/releases)
- [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst)
- [Commits](https://github.com/boto/boto3/compare/1.26.9...1.26.10)

---
updated-dependencies:
- dependency-name: boto3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 4a49eb73..9ea58610 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 -r requirements-apps-api.txt
-boto3==1.26.9
+boto3==1.26.10
 cfn-lint==0.71.1
 flake8==5.0.4
 pypgstac[psycopg]==0.6.10

From b2fe7537b5af1e1c8ca641047f0f9805d9a21d59 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 09:29:31 -0900
Subject: [PATCH 102/144] add debug commands to buildspec

---
 apps/database/buildspec.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index b4194708..cf71911d 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -10,4 +10,12 @@ phases:
       - make install-pypgstac
   build:
     commands:
+      - which python
+      - which pip
+      - python --version
+      - pip --version
+      - which pip3
+      - pip3 --version
+      - echo $PYTHONPATH
+      - echo $PATH
       - make configure-database db_host=$PGHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD

From 5fba7268b2e8924893480ce12117067b2d12e425 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 09:35:47 -0900
Subject: [PATCH 103/144] more debugging commands

---
 apps/database/buildspec.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index cf71911d..93cdd580 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -18,4 +18,6 @@ phases:
       - pip3 --version
       - echo $PYTHONPATH
       - echo $PATH
+      - ls -l /root/.pyenv/bin
+      - find / -name pypgstac
       - make configure-database db_host=$PGHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD

From bf131e1b4da0b0453959349f73cfad9d8ebda6f8 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 09:47:29 -0900
Subject: [PATCH 104/144] more debugging

---
 apps/database/buildspec.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index 93cdd580..965e1ef0 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -19,5 +19,6 @@ phases:
       - echo $PYTHONPATH
       - echo $PATH
       - ls -l /root/.pyenv/bin
-      - find / -name pypgstac
+      - find /root/.pyenv/ -name pypgstac
+      - pyenv rehash
       - make configure-database db_host=$PGHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD

From bdeb4874ae2dfb7da2fff2a9f7b9a0583d127d7c Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 09:51:45 -0900
Subject: [PATCH 105/144] working buildspec

---
 apps/database/buildspec.yml | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/apps/database/buildspec.yml b/apps/database/buildspec.yml
index 965e1ef0..c3b03f31 100644
--- a/apps/database/buildspec.yml
+++ b/apps/database/buildspec.yml
@@ -8,17 +8,7 @@ phases:
       - apt update
       - apt install -y postgresql-12
       - make install-pypgstac
+      - pyenv rehash
   build:
     commands:
-      - which python
-      - which pip
-      - python --version
-      - pip --version
-      - which pip3
-      - pip3 --version
-      - echo $PYTHONPATH
-      - echo $PATH
-      - ls -l /root/.pyenv/bin
-      - find /root/.pyenv/ -name pypgstac
-      - pyenv rehash
       - make configure-database db_host=$PGHOST db_admin_password=$PGPASSWORD db_read_password=$READ_PASSWORD

From 402bca99dc6bae2e1301bc0856b434094b9201be Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 10:49:01 -0900
Subject: [PATCH 106/144] move database and api to public subnets

---
 apps/cloudformation.yml          | 5 +++--
 apps/database/cloudformation.yml | 9 ++++++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 31687952..8f4b045a 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -37,7 +37,7 @@ Resources:
         DatabaseHost: !GetAtt Database.Outputs.DatabaseHost
         DatabaseReadPassword: !Ref DatabaseReadPassword
         SecurityGroupId: !GetAtt Database.Outputs.ClientSecurityGroupId
-        SubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
+        SubnetIds: !GetAtt VPC.Outputs.PublicSubnets
       TemplateURL: api/cloudformation.yml
 
   Database:
@@ -47,7 +47,8 @@ Resources:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
         DatabaseReadPassword: !Ref DatabaseReadPassword
         VpcId: !GetAtt VPC.Outputs.VPC
-        SubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
+        PublicSubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
+        PrivateSubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
         CidrIp: !Ref CidrIp
         GithubBranch: !Ref GithubBranch
       TemplateURL: database/cloudformation.yml
diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 0ad94976..385d048c 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -13,7 +13,10 @@ Parameters:
   VpcId:
     Type: AWS::EC2::VPC::Id
 
-  SubnetIds:
+  PublicSubnetIds:
+    Type: CommaDelimitedList
+
+  PrivateSubnetIds:
     Type: CommaDelimitedList
 
   CidrIp:
@@ -57,7 +60,7 @@ Resources:
     Type: AWS::RDS::DBSubnetGroup
     Properties:
       DBSubnetGroupDescription: !Sub "Subnet Group for ${AWS::StackName} database"
-      SubnetIds: !Ref SubnetIds
+      SubnetIds: !Ref PublicSubnetIds
 
   DatabaseSecurityGroup:
     Type: AWS::EC2::SecurityGroup
@@ -115,7 +118,7 @@ Resources:
         Type: NO_ARTIFACTS
       VpcConfig:
         VpcId: !Ref VpcId
-        Subnets: !Ref SubnetIds
+        Subnets: !Ref PrivateSubnetIds
         SecurityGroupIds:
           - !Ref ClientSecurityGroup
 

From 711439a688026fffd7dbe12c84878417c931e0f6 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 16 Nov 2022 11:11:15 -0900
Subject: [PATCH 107/144] Wait for CodeBuild in deploy workflow

---
 .github/workflows/deploy-stac-api.yml |  2 +-
 wait_for_codebuild.py                 | 35 +++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 wait_for_codebuild.py

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 2cd462fd..1c5b9086 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -49,4 +49,4 @@ jobs:
                                     --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
                                     --output text)" >> $GITHUB_ENV
       - name: Run CodeBuild
-        run: aws codebuild start-build --project-name $CODEBUILD_PROJECT
+        run: python wait_for_codebuild.py
diff --git a/wait_for_codebuild.py b/wait_for_codebuild.py
new file mode 100644
index 00000000..ff59412b
--- /dev/null
+++ b/wait_for_codebuild.py
@@ -0,0 +1,35 @@
+import os
+import sys
+import time
+
+import boto3
+
+CLIENT = boto3.client('codebuild')
+
+
+def main() -> None:
+    project_name = os.environ['CODEBUILD_PROJECT']
+    print(f'Starting CodeBuild for project {project_name}')
+    response = CLIENT.start_build(projectName=project_name)
+
+    build_id = response['build']['id']
+    print(f'Build ID: {build_id}')
+
+    build_status = response['build']['buildStatus']
+    print(f'Build status: {build_status}')
+
+    while build_status == 'IN_PROGRESS':
+        time.sleep(5)
+
+        response = CLIENT.batch_get_builds(ids=[build_id])
+        assert len(response['builds']) == 1
+
+        build_status = response['builds'][0]['buildStatus']
+        print(f'Build status: {build_status}')
+
+    if build_status != 'SUCCEEDED':
+        sys.exit(f'CodeBuild failed with status {build_status}')
+
+
+if __name__ == '__main__':
+    main()

From 2a5333a301e46cd1d332fa8040b6ad5bda6725ef Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Wed, 16 Nov 2022 11:41:11 -0900
Subject: [PATCH 108/144] rename codebuild script

---
 .github/workflows/deploy-stac-api.yml     | 2 +-
 wait_for_codebuild.py => run_codebuild.py | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename wait_for_codebuild.py => run_codebuild.py (100%)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 1c5b9086..15395c58 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -49,4 +49,4 @@ jobs:
                                     --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
                                     --output text)" >> $GITHUB_ENV
       - name: Run CodeBuild
-        run: python wait_for_codebuild.py
+        run: python run_codebuild.py
diff --git a/wait_for_codebuild.py b/run_codebuild.py
similarity index 100%
rename from wait_for_codebuild.py
rename to run_codebuild.py

From 06dc70da091e8129c03cb47834517e402de7e76f Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 14:45:55 -0900
Subject: [PATCH 109/144] set github branch in test deployment workflow

---
 .github/workflows/deploy-stac-api.yml | 4 +++-
 Makefile                              | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 2cd462fd..07f25ad3 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -13,6 +13,7 @@ jobs:
         include:
           - environment: asf-stac-test
             template_bucket: cf-templates-aubvn3i9olmk-us-west-2
+            github_branch: develop
 
     environment:
       name: ${{ matrix.environment }}
@@ -41,7 +42,8 @@ jobs:
             db_read_password=${{ secrets.DB_READ_PASSWORD }} \
             vpc_id=${{ secrets.VPC_ID }} \
             subnet_ids=${{ secrets.SUBNET_IDS }} \
-            cidr_ip=${{ secrets.CIDR_IP }}
+            cidr_ip=${{ secrets.CIDR_IP }} \
+            github_branch=${{ matrix.github_branch }}
       - name: Get CodeBuild project
         run: |
           echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
diff --git a/Makefile b/Makefile
index 7cb3fc89..8789b912 100644
--- a/Makefile
+++ b/Makefile
@@ -25,7 +25,8 @@ deploy:
 	      DatabaseReadPassword=${db_read_password} \
 	      VpcId=${vpc_id} \
 	      SubnetIds=${subnet_ids} \
-	      CidrIp=${cidr_ip}
+	      CidrIp=${cidr_ip} \
+	      GithubBranch=${github_branch}
 
 psql:
 	PGHOST=${db_host} PGPORT=5432 PGDATABASE=postgres PGUSER=${db_user} PGPASSWORD=${db_password} psql

From 0dcd9ee3b27b20170b8fb98d65384bb2f7cdfc9b Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 14:47:56 -0900
Subject: [PATCH 110/144] fix subnet typo in cloudformation.yml

---
 apps/cloudformation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/cloudformation.yml b/apps/cloudformation.yml
index 8f4b045a..3b457da1 100644
--- a/apps/cloudformation.yml
+++ b/apps/cloudformation.yml
@@ -47,7 +47,7 @@ Resources:
         DatabaseAdminPassword: !Ref DatabaseAdminPassword
         DatabaseReadPassword: !Ref DatabaseReadPassword
         VpcId: !GetAtt VPC.Outputs.VPC
-        PublicSubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
+        PublicSubnetIds: !GetAtt VPC.Outputs.PublicSubnets
         PrivateSubnetIds: !GetAtt VPC.Outputs.PrivateSubnets
         CidrIp: !Ref CidrIp
         GithubBranch: !Ref GithubBranch

From a1ebbc590cf5ba0749082ddeaf3636aa1b3ce3d4 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 14:59:13 -0900
Subject: [PATCH 111/144] logging permissions, log group

---
 apps/database/cloudformation.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index 385d048c..ec10699f 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -106,7 +106,6 @@ Resources:
           - Name: READ_PASSWORD
             Type: SECRETS_MANAGER
             Value: !Sub "${DatabaseSecret}:read_password"
-#      LogsConfig:
       ServiceRole: !Ref CodeBuildServiceRole
       Source:
         Type: GITHUB
@@ -122,6 +121,12 @@ Resources:
         SecurityGroupIds:
           - !Ref ClientSecurityGroup
 
+  CodeBuildLogGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Sub "/aws/codebuild/${CodeBuildProject}"
+      RetentionInDays: 90
+
   CodeBuildServiceRole:
     Type: AWS::IAM::Role
     Properties:
@@ -142,12 +147,9 @@ Resources:
                   - logs:CreateLogGroup
                   - logs:CreateLogStream
                   - logs:PutLogEvents
-                  - codecommit:GitPull
-                  - s3:GetObject
-                  - s3:GetObjectVersion
-                  - s3:PutObject
-                  - s3:GetBucketAcl
-                  - s3:GetBucketLocation
+                Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/*"
+              - Effect: Allow
+                Action:
                   - ec2:CreateNetworkInterface
                   - ec2:DescribeDhcpOptions
                   - ec2:DescribeNetworkInterfaces

From 0b44430dae36a85792cac1116a8fa514c9a5d4b3 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 15:02:52 -0900
Subject: [PATCH 112/144] add link to source of vpc cloudformation template

---
 apps/vpc/cloudformation.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/vpc/cloudformation.yml b/apps/vpc/cloudformation.yml
index 28193f9f..6f3af552 100644
--- a/apps/vpc/cloudformation.yml
+++ b/apps/vpc/cloudformation.yml
@@ -1,3 +1,4 @@
+# https://docs.aws.amazon.com/codebuild/latest/userguide/cloudformation-vpc-template.html
 Description:  This template deploys a VPC, with a pair of public and private subnets spread
   across two Availability Zones. It deploys an internet gateway, with a default
   route on the public subnets. It deploys a pair of NAT gateways (one in each AZ),

From 557300c78eef398238a032852ae247b6d057e7bd Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 15:12:39 -0900
Subject: [PATCH 113/144] drop createloggroup permissions from codebuild
 project

---
 apps/database/cloudformation.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index ec10699f..e11237e7 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -144,7 +144,6 @@ Resources:
             Statement:
               - Effect: Allow
                 Action:
-                  - logs:CreateLogGroup
                   - logs:CreateLogStream
                   - logs:PutLogEvents
                 Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/*"

From 3d4c83a6938d76ff846921ecde341d17b202c08f Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 15:22:58 -0900
Subject: [PATCH 114/144] drop obsolete vpc/subnet parameters from makefile and
 deploy workflow

---
 .github/workflows/deploy-stac-api.yml | 2 --
 Makefile                              | 2 --
 2 files changed, 4 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 9532fbab..77b49084 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -40,8 +40,6 @@ jobs:
             cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
             db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
             db_read_password=${{ secrets.DB_READ_PASSWORD }} \
-            vpc_id=${{ secrets.VPC_ID }} \
-            subnet_ids=${{ secrets.SUBNET_IDS }} \
             cidr_ip=${{ secrets.CIDR_IP }} \
             github_branch=${{ matrix.github_branch }}
       - name: Get CodeBuild project
diff --git a/Makefile b/Makefile
index 8789b912..95bc1894 100644
--- a/Makefile
+++ b/Makefile
@@ -23,8 +23,6 @@ deploy:
 	    --parameter-overrides \
 	      DatabaseAdminPassword=${db_admin_password} \
 	      DatabaseReadPassword=${db_read_password} \
-	      VpcId=${vpc_id} \
-	      SubnetIds=${subnet_ids} \
 	      CidrIp=${cidr_ip} \
 	      GithubBranch=${github_branch}
 

From 834a4ca9b39e5d65d65c597d7e06008ef921b77d Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Wed, 16 Nov 2022 19:03:17 -0900
Subject: [PATCH 115/144] Update apps/database/cloudformation.yml

Co-authored-by: Jake Herrmann <jtherrmann@alaska.edu>
---
 apps/database/cloudformation.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/database/cloudformation.yml b/apps/database/cloudformation.yml
index e11237e7..c5b76c9f 100644
--- a/apps/database/cloudformation.yml
+++ b/apps/database/cloudformation.yml
@@ -65,7 +65,7 @@ Resources:
   DatabaseSecurityGroup:
     Type: AWS::EC2::SecurityGroup
     Properties:
-      GroupDescription: !Sub "Security group for clients of ${AWS::StackName} database"
+      GroupDescription: !Sub "Security group for ${AWS::StackName} database"
       VpcId: !Ref VpcId
       SecurityGroupIngress:
         - IpProtocol: tcp

From 89fcda381284f64fbd3289c3a9afea562d3b9d07 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 09:42:39 -0900
Subject: [PATCH 116/144] Update README

---
 README.md | 64 ++++++++++++++++++++++++++++---------------------------
 1 file changed, 33 insertions(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 1b1372e1..6903fbd6 100644
--- a/README.md
+++ b/README.md
@@ -1,17 +1,41 @@
 # asf-stac
 
-A repository containing code related to the creation and hosting of STAC catalogs by the ASF tools team.
+A repository containing code related to the creation and hosting of STAC catalogs by the ASF Tools team.
 
 ## Developer setup
 
-TODO: document creating conda env and installing developer deps
+Create the environment and install developer dependencies:
 
-## STAC API
+```
+conda create -n asf-stac python=3.9
+conda activate asf-stac
+make install
+```
 
-TODO: proofread docs since adding multiple database users
+## STAC API
 
 TODO: document database URLs and the `/api.html` endpoint for the Swagger UI
 
+### Running the API locally
+
+You can run the STAC API frontend locally (connected to the AWS-hosted database). This is required for accessing
+the create/update/delete endpoints (which are provided by the STAC API's Transaction extension), as these
+endpoints are disabled for the publicly accessible API.
+
+To run the STAC API locally:
+
+```
+make run-api db_host=<host> db_admin_password=<password>
+```
+
+You should see something like `Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)` in the output; you can
+query the API at that URL.
+
+### Ingesting a STAC dataset
+
+Run `python ingest_data.py -h` for usage instructions. The script only works with an API that supports the
+Transaction extension. See [Running the API locally](#running-the-api-locally).
+
 ### Upgrading the database
 
 The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs
@@ -36,39 +60,17 @@ The initial AWS deployment creates a Postgres database, installs the PostGIS ext
 
 5. Deploy to AWS.
 
-PgSTAC upgrades are handled automatically: the [deployment workflow](.github/workflows/deploy-stac-api.yml)
-migrates the database to the installed version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac>
-for more information about migrations.
-
-### Retrieving database connection details
-
-The database host and database user credentials are available via the AWS Secrets Manager console
-in the AWS account where the CloudFormation stack was deployed.
+PgSTAC upgrades are handled automatically: the deployment pipeline migrates the database to the installed
+version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac> for more information about migrations.
 
 ### Connecting to the database
 
+We shouldn't need to manually connect to the database, but we can if we need to.
 Confirm you have the `psql` command installed, then run:
 
 ```
 make psql db_host=<host> db_user=<user> db_password=<password>
 ```
 
-### Running the API locally
-
-You can run the STAC API frontend locally (connected to the AWS-hosted database). This is required for accessing
-the create/update/delete endpoints (which are provided by the STAC API's Transaction extension), as these
-endpoints are disabled for the publicly accessible API.
-
-To run the STAC API locally:
-
-```
-make run-api db_host=<host> db_admin_password=<password>
-```
-
-You should see something like `Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)` in the output; you can
-query the API at that URL.
-
-### Ingesting STAC dataset
-
-Run `python ingest_data.py -h` for usage instructions. The script only works with an API that supports the
-Transaction extension.
+The database host and database user credentials are available via the AWS Secrets Manager console
+in the AWS account where the CloudFormation stack was deployed.

From 2c812dde937a9e5094f55c9384d43c0d7927a1d7 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 10:12:40 -0900
Subject: [PATCH 117/144] Add db client restrictions to README

---
 README.md | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6903fbd6..d4124cc3 100644
--- a/README.md
+++ b/README.md
@@ -22,7 +22,8 @@ You can run the STAC API frontend locally (connected to the AWS-hosted database)
 the create/update/delete endpoints (which are provided by the STAC API's Transaction extension), as these
 endpoints are disabled for the publicly accessible API.
 
-To run the STAC API locally:
+Confirm that you're working from a machine with access to the database.
+See [Connecting to the database](#connecting-to-the-database) for exact requirements. Then run:
 
 ```
 make run-api db_host=<host> db_admin_password=<password>
@@ -66,6 +67,11 @@ version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac> for mo
 ### Connecting to the database
 
 We shouldn't need to manually connect to the database, but we can if we need to.
+
+The database only accepts connections from within the ASF network or from clients
+with the client security group attached. See the ingress rules for the database security group in the
+[database CloudFormation template](apps/database/cloudformation.yml).
+
 Confirm you have the `psql` command installed, then run:
 
 ```

From ae0ab8c681f45f7b829fbd823370146c0dc68975 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Thu, 17 Nov 2022 10:54:17 -0900
Subject: [PATCH 118/144] don't need to install pypgstac in github workflow

---
 .github/workflows/deploy-stac-api.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 2ea1c59a..0f4ce4d2 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -31,8 +31,6 @@ jobs:
           python-version: 3.9
       - name: Install Lambda dependencies
         run: make -s install-lambda-deps
-      - name: Install pyPgSTAC
-        run: make -s install-pypgstac
       - name: Deploy CloudFormation stack
         run: |
           make -s deploy \

From b985cc52360b2b6588e2dc34177cda5773e3946a Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Thu, 17 Nov 2022 12:07:04 -0900
Subject: [PATCH 119/144] add conditional deployment to github workflow

---
 .github/workflows/deploy-stac-api.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 0f4ce4d2..c077727d 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -8,10 +8,12 @@ on:
 jobs:
   deploy-stac-api:
     runs-on: ubuntu-latest
+    if: github.ref == matrix.deploy_ref
     strategy:
       matrix:
         include:
           - environment: asf-stac-test
+            deploy_ref: refs/heads/develop
             template_bucket: cf-templates-aubvn3i9olmk-us-west-2
             github_branch: develop
             domain_name: stac-test.asf.alaska.edu

From 7dc7222e51e853f9a004571e127cadc2d097f2ab Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Thu, 17 Nov 2022 12:12:00 -0900
Subject: [PATCH 120/144] install boto3 for run_codebuild.py in github
 deployment workflow

---
 .github/workflows/deploy-stac-api.yml | 2 ++
 requirements-run-codebuild.txt        | 1 +
 requirements.txt                      | 1 +
 3 files changed, 4 insertions(+)
 create mode 100644 requirements-run-codebuild.txt

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 0f4ce4d2..d9a9c0e9 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -31,6 +31,8 @@ jobs:
           python-version: 3.9
       - name: Install Lambda dependencies
         run: make -s install-lambda-deps
+      - name: Install Python dependencies for deployment scripts
+        run: python -m pip install -r requirements-run-codebuild.yml
       - name: Deploy CloudFormation stack
         run: |
           make -s deploy \
diff --git a/requirements-run-codebuild.txt b/requirements-run-codebuild.txt
new file mode 100644
index 00000000..951fc34f
--- /dev/null
+++ b/requirements-run-codebuild.txt
@@ -0,0 +1 @@
+boto3==1.26.10
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 9ea58610..ae5d2774 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,5 @@
 -r requirements-apps-api.txt
+-r requirements-run-codebuild.txt
 boto3==1.26.10
 cfn-lint==0.71.1
 flake8==5.0.4

From 7edbdec1067dc554eb909bd9c0b4cf6f247c94eb Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Thu, 17 Nov 2022 13:01:19 -0900
Subject: [PATCH 121/144] Update .github/workflows/deploy-stac-api.yml

Co-authored-by: Joseph H Kennedy <me@jhkennedy.org>
---
 .github/workflows/deploy-stac-api.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index d9a9c0e9..328c7986 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -32,7 +32,7 @@ jobs:
       - name: Install Lambda dependencies
         run: make -s install-lambda-deps
       - name: Install Python dependencies for deployment scripts
-        run: python -m pip install -r requirements-run-codebuild.yml
+        run: python -m pip install -r requirements-run-codebuild.txt
       - name: Deploy CloudFormation stack
         run: |
           make -s deploy \

From e5bd8abda9205a33b44ad0328a7184df509dbf01 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 17 Nov 2022 22:08:33 +0000
Subject: [PATCH 122/144] Bump boto3 from 1.26.10 to 1.26.12

Bumps [boto3](https://github.com/boto/boto3) from 1.26.10 to 1.26.12.
- [Release notes](https://github.com/boto/boto3/releases)
- [Changelog](https://github.com/boto/boto3/blob/develop/CHANGELOG.rst)
- [Commits](https://github.com/boto/boto3/compare/1.26.10...1.26.12)

---
updated-dependencies:
- dependency-name: boto3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 requirements-run-codebuild.txt | 2 +-
 requirements.txt               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements-run-codebuild.txt b/requirements-run-codebuild.txt
index 951fc34f..c366ded5 100644
--- a/requirements-run-codebuild.txt
+++ b/requirements-run-codebuild.txt
@@ -1 +1 @@
-boto3==1.26.10
\ No newline at end of file
+boto3==1.26.12
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index ae5d2774..3265b9ce 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 -r requirements-apps-api.txt
 -r requirements-run-codebuild.txt
-boto3==1.26.10
+boto3==1.26.12
 cfn-lint==0.71.1
 flake8==5.0.4
 pypgstac[psycopg]==0.6.10

From 5eb410d3feb7c1b40de91482cd75612410b19f87 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 13:21:52 -0900
Subject: [PATCH 123/144] add api url to readme

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d4124cc3..c59a493e 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,10 @@ make install
 
 ## STAC API
 
-TODO: document database URLs and the `/api.html` endpoint for the Swagger UI
+The test API is available at <https://stac-test.asf.alaska.edu>
+and the Swagger UI is available at <https://stac-test.asf.alaska.edu/api.html>.
+
+TODO: document prod URL
 
 ### Running the API locally
 

From 6820283da7d482ae627cce0c0c65052bf9796b40 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Thu, 17 Nov 2022 13:24:25 -0900
Subject: [PATCH 124/144] add production entry to deployment matrix

---
 .github/workflows/deploy-stac-api.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index fad783a4..6fed4f04 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -18,6 +18,12 @@ jobs:
             github_branch: develop
             domain_name: stac-test.asf.alaska.edu
 
+          - environment: asf-stac
+            deploy_ref: refs/heads/main
+            template_bucket: cf-templates-aubvn3i9olmk-us-west-2
+            github_branch: main
+            domain_name: stac.asf.alaska.edu
+
     environment:
       name: ${{ matrix.environment }}
 

From daee9fecae8fd39dc12cb7ccae15135a4979a572 Mon Sep 17 00:00:00 2001
From: Andrew Johnston <asjohnston@alaska.edu>
Date: Thu, 17 Nov 2022 14:04:04 -0900
Subject: [PATCH 125/144] add main to triggers for deployment workflow

---
 .github/workflows/deploy-stac-api.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
index 6fed4f04..e04a9d1c 100644
--- a/.github/workflows/deploy-stac-api.yml
+++ b/.github/workflows/deploy-stac-api.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - develop
+      - main
 
 jobs:
   deploy-stac-api:

From 2b68af3607346f100eca1a2d638fba8e31fb5c6e Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:29:38 -0900
Subject: [PATCH 126/144] Add reusable deploy action

---
 .github/actions/deploy-stac.yml        | 49 ++++++++++++++++++++
 .github/workflows/deploy-stac-api.yml  | 64 --------------------------
 .github/workflows/deploy-stac-prod.yml | 26 +++++++++++
 .github/workflows/deploy-stac-test.yml | 20 ++++++++
 4 files changed, 95 insertions(+), 64 deletions(-)
 create mode 100644 .github/actions/deploy-stac.yml
 delete mode 100644 .github/workflows/deploy-stac-api.yml
 create mode 100644 .github/workflows/deploy-stac-prod.yml
 create mode 100644 .github/workflows/deploy-stac-test.yml

diff --git a/.github/actions/deploy-stac.yml b/.github/actions/deploy-stac.yml
new file mode 100644
index 00000000..b5c834d1
--- /dev/null
+++ b/.github/actions/deploy-stac.yml
@@ -0,0 +1,49 @@
+name: Deploy STAC API
+description: Deploy the STAC API
+
+inputs:
+  STACK_NAME:
+    required: true
+  GITHUB_BRANCH:
+    required: true
+  DOMAIN_NAME:
+    required: true
+  CF_TEMPLATE_BUCKET:
+    default: cf-templates-aubvn3i9olmk-us-west-2  # For HyP3 account
+
+runs:
+  using: composite
+  steps:
+      - uses: actions/checkout@v3
+        - uses: aws-actions/configure-aws-credentials@v1
+          with:
+            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            aws-region: us-west-2
+        - uses: actions/setup-python@v4
+          with:
+            python-version: 3.9
+        - name: Install Lambda dependencies
+          run: make -s install-lambda-deps
+        - name: Install Python dependencies for deployment scripts
+          run: python -m pip install -r requirements-run-codebuild.txt
+        - name: Deploy CloudFormation stack
+          run: |
+            make -s deploy \
+              stack_name=${{ inputs.STACK_NAME }} \
+              s3_bucket=${{ inputs.CF_TEMPLATE_BUCKET }} \
+              cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
+              db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
+              db_read_password=${{ secrets.DB_READ_PASSWORD }} \
+              cidr_ip=${{ secrets.CIDR_IP }} \
+              github_branch=${{ inputs.GITHUB_BRANCH }} \
+              domain_name=${{ inputs.DOMAIN_NAME }} \
+              certificate_arn=${{ secrets.CERTIFICATE_ARN }}
+        - name: Get CodeBuild project
+          run: |
+            echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
+                                      --stack-name ${{ inputs.STACK_NAME }} \
+                                      --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
+                                      --output text)" >> $GITHUB_ENV
+        - name: Run CodeBuild
+          run: python run_codebuild.py
diff --git a/.github/workflows/deploy-stac-api.yml b/.github/workflows/deploy-stac-api.yml
deleted file mode 100644
index e04a9d1c..00000000
--- a/.github/workflows/deploy-stac-api.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-name: Deploy STAC API
-
-on:
-  push:
-    branches:
-      - develop
-      - main
-
-jobs:
-  deploy-stac-api:
-    runs-on: ubuntu-latest
-    if: github.ref == matrix.deploy_ref
-    strategy:
-      matrix:
-        include:
-          - environment: asf-stac-test
-            deploy_ref: refs/heads/develop
-            template_bucket: cf-templates-aubvn3i9olmk-us-west-2
-            github_branch: develop
-            domain_name: stac-test.asf.alaska.edu
-
-          - environment: asf-stac
-            deploy_ref: refs/heads/main
-            template_bucket: cf-templates-aubvn3i9olmk-us-west-2
-            github_branch: main
-            domain_name: stac.asf.alaska.edu
-
-    environment:
-      name: ${{ matrix.environment }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-      - name: Install Lambda dependencies
-        run: make -s install-lambda-deps
-      - name: Install Python dependencies for deployment scripts
-        run: python -m pip install -r requirements-run-codebuild.txt
-      - name: Deploy CloudFormation stack
-        run: |
-          make -s deploy \
-            stack_name=${{ matrix.environment }} \
-            s3_bucket=${{ matrix.template_bucket }} \
-            cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
-            db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
-            db_read_password=${{ secrets.DB_READ_PASSWORD }} \
-            cidr_ip=${{ secrets.CIDR_IP }} \
-            github_branch=${{ matrix.github_branch }} \
-            domain_name=${{ matrix.domain_name }} \
-            certificate_arn=${{ secrets.CERTIFICATE_ARN }}
-      - name: Get CodeBuild project
-        run: |
-          echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
-                                    --stack-name ${{ matrix.environment }} \
-                                    --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
-                                    --output text)" >> $GITHUB_ENV
-      - name: Run CodeBuild
-        run: python run_codebuild.py
diff --git a/.github/workflows/deploy-stac-prod.yml b/.github/workflows/deploy-stac-prod.yml
new file mode 100644
index 00000000..ec1870c7
--- /dev/null
+++ b/.github/workflows/deploy-stac-prod.yml
@@ -0,0 +1,26 @@
+name: Deploy STAC API prod
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy-stac-api:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: asf-stac
+
+    steps:
+      - uses: ./.github/actions/deploy-stac
+        with:
+          STACK_NAME: asf-stac
+          GITHUB_BRANCH: main
+          DOMAIN_NAME: stac.asf.alaska.edu
+
+  call-bump-version-workflow:
+    needs: deploy-stac-api
+    uses: ASFHyP3/actions/.github/workflows/reusable-bump-version.yml@v0.4.0
+    secrets:
+      USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
diff --git a/.github/workflows/deploy-stac-test.yml b/.github/workflows/deploy-stac-test.yml
new file mode 100644
index 00000000..79e89474
--- /dev/null
+++ b/.github/workflows/deploy-stac-test.yml
@@ -0,0 +1,20 @@
+name: Deploy STAC API test
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  deploy-stac-api:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: asf-stac-test
+
+    steps:
+      - uses: ./.github/actions/deploy-stac
+        with:
+          STACK_NAME: asf-stac-test
+          GITHUB_BRANCH: develop
+          DOMAIN_NAME: stac-test.asf.alaska.edu

From ba350728033d28ad46ed271e635d6e865030046c Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:31:45 -0900
Subject: [PATCH 127/144] fix indentation

---
 .github/actions/deploy-stac.yml | 64 ++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/.github/actions/deploy-stac.yml b/.github/actions/deploy-stac.yml
index b5c834d1..d9101582 100644
--- a/.github/actions/deploy-stac.yml
+++ b/.github/actions/deploy-stac.yml
@@ -15,35 +15,35 @@ runs:
   using: composite
   steps:
       - uses: actions/checkout@v3
-        - uses: aws-actions/configure-aws-credentials@v1
-          with:
-            aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-            aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-            aws-region: us-west-2
-        - uses: actions/setup-python@v4
-          with:
-            python-version: 3.9
-        - name: Install Lambda dependencies
-          run: make -s install-lambda-deps
-        - name: Install Python dependencies for deployment scripts
-          run: python -m pip install -r requirements-run-codebuild.txt
-        - name: Deploy CloudFormation stack
-          run: |
-            make -s deploy \
-              stack_name=${{ inputs.STACK_NAME }} \
-              s3_bucket=${{ inputs.CF_TEMPLATE_BUCKET }} \
-              cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
-              db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
-              db_read_password=${{ secrets.DB_READ_PASSWORD }} \
-              cidr_ip=${{ secrets.CIDR_IP }} \
-              github_branch=${{ inputs.GITHUB_BRANCH }} \
-              domain_name=${{ inputs.DOMAIN_NAME }} \
-              certificate_arn=${{ secrets.CERTIFICATE_ARN }}
-        - name: Get CodeBuild project
-          run: |
-            echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
-                                      --stack-name ${{ inputs.STACK_NAME }} \
-                                      --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
-                                      --output text)" >> $GITHUB_ENV
-        - name: Run CodeBuild
-          run: python run_codebuild.py
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Install Lambda dependencies
+        run: make -s install-lambda-deps
+      - name: Install Python dependencies for deployment scripts
+        run: python -m pip install -r requirements-run-codebuild.txt
+      - name: Deploy CloudFormation stack
+        run: |
+          make -s deploy \
+            stack_name=${{ inputs.STACK_NAME }} \
+            s3_bucket=${{ inputs.CF_TEMPLATE_BUCKET }} \
+            cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
+            db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
+            db_read_password=${{ secrets.DB_READ_PASSWORD }} \
+            cidr_ip=${{ secrets.CIDR_IP }} \
+            github_branch=${{ inputs.GITHUB_BRANCH }} \
+            domain_name=${{ inputs.DOMAIN_NAME }} \
+            certificate_arn=${{ secrets.CERTIFICATE_ARN }}
+      - name: Get CodeBuild project
+        run: |
+          echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
+                                    --stack-name ${{ inputs.STACK_NAME }} \
+                                    --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
+                                    --output text)" >> $GITHUB_ENV
+      - name: Run CodeBuild
+        run: python run_codebuild.py

From b2db84216af34d3d81a82be981c2b2f3ed6db794 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:32:44 -0900
Subject: [PATCH 128/144] fix indentation again

---
 .github/actions/deploy-stac.yml | 66 ++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/.github/actions/deploy-stac.yml b/.github/actions/deploy-stac.yml
index d9101582..aa3b6f6a 100644
--- a/.github/actions/deploy-stac.yml
+++ b/.github/actions/deploy-stac.yml
@@ -14,36 +14,36 @@ inputs:
 runs:
   using: composite
   steps:
-      - uses: actions/checkout@v3
-      - uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-west-2
-      - uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-      - name: Install Lambda dependencies
-        run: make -s install-lambda-deps
-      - name: Install Python dependencies for deployment scripts
-        run: python -m pip install -r requirements-run-codebuild.txt
-      - name: Deploy CloudFormation stack
-        run: |
-          make -s deploy \
-            stack_name=${{ inputs.STACK_NAME }} \
-            s3_bucket=${{ inputs.CF_TEMPLATE_BUCKET }} \
-            cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
-            db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
-            db_read_password=${{ secrets.DB_READ_PASSWORD }} \
-            cidr_ip=${{ secrets.CIDR_IP }} \
-            github_branch=${{ inputs.GITHUB_BRANCH }} \
-            domain_name=${{ inputs.DOMAIN_NAME }} \
-            certificate_arn=${{ secrets.CERTIFICATE_ARN }}
-      - name: Get CodeBuild project
-        run: |
-          echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
-                                    --stack-name ${{ inputs.STACK_NAME }} \
-                                    --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
-                                    --output text)" >> $GITHUB_ENV
-      - name: Run CodeBuild
-        run: python run_codebuild.py
+    - uses: actions/checkout@v3
+    - uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: us-west-2
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.9
+    - name: Install Lambda dependencies
+      run: make -s install-lambda-deps
+    - name: Install Python dependencies for deployment scripts
+      run: python -m pip install -r requirements-run-codebuild.txt
+    - name: Deploy CloudFormation stack
+      run: |
+        make -s deploy \
+          stack_name=${{ inputs.STACK_NAME }} \
+          s3_bucket=${{ inputs.CF_TEMPLATE_BUCKET }} \
+          cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
+          db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
+          db_read_password=${{ secrets.DB_READ_PASSWORD }} \
+          cidr_ip=${{ secrets.CIDR_IP }} \
+          github_branch=${{ inputs.GITHUB_BRANCH }} \
+          domain_name=${{ inputs.DOMAIN_NAME }} \
+          certificate_arn=${{ secrets.CERTIFICATE_ARN }}
+    - name: Get CodeBuild project
+      run: |
+        echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
+                                  --stack-name ${{ inputs.STACK_NAME }} \
+                                  --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
+                                  --output text)" >> $GITHUB_ENV
+    - name: Run CodeBuild
+      run: python run_codebuild.py

From 1bd3059a3e24204ce440cd14c5b98c262eba1ad3 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:34:16 -0900
Subject: [PATCH 129/144] rename a step

---
 .github/actions/deploy-stac.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/deploy-stac.yml b/.github/actions/deploy-stac.yml
index aa3b6f6a..259f092c 100644
--- a/.github/actions/deploy-stac.yml
+++ b/.github/actions/deploy-stac.yml
@@ -25,7 +25,7 @@ runs:
         python-version: 3.9
     - name: Install Lambda dependencies
       run: make -s install-lambda-deps
-    - name: Install Python dependencies for deployment scripts
+    - name: Install CodeBuild dependencies
       run: python -m pip install -r requirements-run-codebuild.txt
     - name: Deploy CloudFormation stack
       run: |

From 8086467226fda6ed584ec7bc25bb24838b9f4676 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:38:15 -0900
Subject: [PATCH 130/144] move checkout action

---
 .github/actions/deploy-stac.yml        | 1 -
 .github/workflows/deploy-stac-prod.yml | 1 +
 .github/workflows/deploy-stac-test.yml | 1 +
 3 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/actions/deploy-stac.yml b/.github/actions/deploy-stac.yml
index 259f092c..8c120b26 100644
--- a/.github/actions/deploy-stac.yml
+++ b/.github/actions/deploy-stac.yml
@@ -14,7 +14,6 @@ inputs:
 runs:
   using: composite
   steps:
-    - uses: actions/checkout@v3
     - uses: aws-actions/configure-aws-credentials@v1
       with:
         aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/deploy-stac-prod.yml b/.github/workflows/deploy-stac-prod.yml
index ec1870c7..7de25f4c 100644
--- a/.github/workflows/deploy-stac-prod.yml
+++ b/.github/workflows/deploy-stac-prod.yml
@@ -13,6 +13,7 @@ jobs:
       name: asf-stac
 
     steps:
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/deploy-stac
         with:
           STACK_NAME: asf-stac
diff --git a/.github/workflows/deploy-stac-test.yml b/.github/workflows/deploy-stac-test.yml
index 79e89474..d4437b70 100644
--- a/.github/workflows/deploy-stac-test.yml
+++ b/.github/workflows/deploy-stac-test.yml
@@ -13,6 +13,7 @@ jobs:
       name: asf-stac-test
 
     steps:
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/deploy-stac
         with:
           STACK_NAME: asf-stac-test

From 25b24f88df765906ce082081b9b3567c6827e581 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:41:13 -0900
Subject: [PATCH 131/144] fix action

---
 .github/actions/{deploy-stac.yml => deploy-stac/action.yml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/actions/{deploy-stac.yml => deploy-stac/action.yml} (100%)

diff --git a/.github/actions/deploy-stac.yml b/.github/actions/deploy-stac/action.yml
similarity index 100%
rename from .github/actions/deploy-stac.yml
rename to .github/actions/deploy-stac/action.yml

From d12b57479be34df3b6ed9685f712713457e1cd30 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 14:52:19 -0900
Subject: [PATCH 132/144] pass secrets as inputs to reusable deploy action

---
 .github/actions/deploy-stac/action.yml | 28 +++++++++++++++++++-------
 .github/workflows/deploy-stac-prod.yml |  7 +++++++
 .github/workflows/deploy-stac-test.yml |  7 +++++++
 3 files changed, 35 insertions(+), 7 deletions(-)

diff --git a/.github/actions/deploy-stac/action.yml b/.github/actions/deploy-stac/action.yml
index 8c120b26..ef18bbeb 100644
--- a/.github/actions/deploy-stac/action.yml
+++ b/.github/actions/deploy-stac/action.yml
@@ -2,6 +2,20 @@ name: Deploy STAC API
 description: Deploy the STAC API
 
 inputs:
+  AWS_ACCESS_KEY_ID:
+    required: true
+  AWS_SECRET_ACCESS_KEY:
+    required: true
+  CLOUDFORMATION_ROLE_ARN:
+    required: true
+  DB_ADMIN_PASSWORD:
+    required: true
+  DB_READ_PASSWORD:
+    required: true
+  CIDR_IP:
+    required: true
+  CERTIFICATE_ARN:
+    required: true
   STACK_NAME:
     required: true
   GITHUB_BRANCH:
@@ -16,8 +30,8 @@ runs:
   steps:
     - uses: aws-actions/configure-aws-credentials@v1
       with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-access-key-id: ${{ inputs.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ inputs.AWS_SECRET_ACCESS_KEY }}
         aws-region: us-west-2
     - uses: actions/setup-python@v4
       with:
@@ -31,13 +45,13 @@ runs:
         make -s deploy \
           stack_name=${{ inputs.STACK_NAME }} \
           s3_bucket=${{ inputs.CF_TEMPLATE_BUCKET }} \
-          cloudformation_role_arn=${{ secrets.CLOUDFORMATION_ROLE_ARN }} \
-          db_admin_password=${{ secrets.DB_ADMIN_PASSWORD }} \
-          db_read_password=${{ secrets.DB_READ_PASSWORD }} \
-          cidr_ip=${{ secrets.CIDR_IP }} \
+          cloudformation_role_arn=${{ inputs.CLOUDFORMATION_ROLE_ARN }} \
+          db_admin_password=${{ inputs.DB_ADMIN_PASSWORD }} \
+          db_read_password=${{ inputs.DB_READ_PASSWORD }} \
+          cidr_ip=${{ inputs.CIDR_IP }} \
           github_branch=${{ inputs.GITHUB_BRANCH }} \
           domain_name=${{ inputs.DOMAIN_NAME }} \
-          certificate_arn=${{ secrets.CERTIFICATE_ARN }}
+          certificate_arn=${{ inputs.CERTIFICATE_ARN }}
     - name: Get CodeBuild project
       run: |
         echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
diff --git a/.github/workflows/deploy-stac-prod.yml b/.github/workflows/deploy-stac-prod.yml
index 7de25f4c..6924bcb7 100644
--- a/.github/workflows/deploy-stac-prod.yml
+++ b/.github/workflows/deploy-stac-prod.yml
@@ -16,6 +16,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/deploy-stac
         with:
+          AWS_ACCESS_KEY_ID: secrets.AWS_ACCESS_KEY_ID
+          AWS_SECRET_ACCESS_KEY: secrets.AWS_SECRET_ACCESS_KEY
+          CLOUDFORMATION_ROLE_ARN: secrets.CLOUDFORMATION_ROLE_ARN
+          DB_ADMIN_PASSWORD: secrets.DB_ADMIN_PASSWORD
+          DB_READ_PASSWORD: secrets.DB_READ_PASSWORD
+          CIDR_IP: secrets.CIDR_IP
+          CERTIFICATE_ARN: secrets.CERTIFICATE_ARN
           STACK_NAME: asf-stac
           GITHUB_BRANCH: main
           DOMAIN_NAME: stac.asf.alaska.edu
diff --git a/.github/workflows/deploy-stac-test.yml b/.github/workflows/deploy-stac-test.yml
index d4437b70..da718586 100644
--- a/.github/workflows/deploy-stac-test.yml
+++ b/.github/workflows/deploy-stac-test.yml
@@ -16,6 +16,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/deploy-stac
         with:
+          AWS_ACCESS_KEY_ID: secrets.AWS_ACCESS_KEY_ID
+          AWS_SECRET_ACCESS_KEY: secrets.AWS_SECRET_ACCESS_KEY
+          CLOUDFORMATION_ROLE_ARN: secrets.CLOUDFORMATION_ROLE_ARN
+          DB_ADMIN_PASSWORD: secrets.DB_ADMIN_PASSWORD
+          DB_READ_PASSWORD: secrets.DB_READ_PASSWORD
+          CIDR_IP: secrets.CIDR_IP
+          CERTIFICATE_ARN: secrets.CERTIFICATE_ARN
           STACK_NAME: asf-stac-test
           GITHUB_BRANCH: develop
           DOMAIN_NAME: stac-test.asf.alaska.edu

From eba9f66c4bcaf6eed6f478e8eb10978ecf8dfd42 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 15:00:08 -0900
Subject: [PATCH 133/144] fix github secrets syntax

---
 .github/workflows/deploy-stac-prod.yml | 14 +++++++-------
 .github/workflows/deploy-stac-test.yml | 14 +++++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/deploy-stac-prod.yml b/.github/workflows/deploy-stac-prod.yml
index 6924bcb7..c4be785d 100644
--- a/.github/workflows/deploy-stac-prod.yml
+++ b/.github/workflows/deploy-stac-prod.yml
@@ -16,13 +16,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/deploy-stac
         with:
-          AWS_ACCESS_KEY_ID: secrets.AWS_ACCESS_KEY_ID
-          AWS_SECRET_ACCESS_KEY: secrets.AWS_SECRET_ACCESS_KEY
-          CLOUDFORMATION_ROLE_ARN: secrets.CLOUDFORMATION_ROLE_ARN
-          DB_ADMIN_PASSWORD: secrets.DB_ADMIN_PASSWORD
-          DB_READ_PASSWORD: secrets.DB_READ_PASSWORD
-          CIDR_IP: secrets.CIDR_IP
-          CERTIFICATE_ARN: secrets.CERTIFICATE_ARN
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
+          DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
+          DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
+          CIDR_IP: ${{ secrets.CIDR_IP }}
+          CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }}
           STACK_NAME: asf-stac
           GITHUB_BRANCH: main
           DOMAIN_NAME: stac.asf.alaska.edu
diff --git a/.github/workflows/deploy-stac-test.yml b/.github/workflows/deploy-stac-test.yml
index da718586..1221b33b 100644
--- a/.github/workflows/deploy-stac-test.yml
+++ b/.github/workflows/deploy-stac-test.yml
@@ -16,13 +16,13 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/deploy-stac
         with:
-          AWS_ACCESS_KEY_ID: secrets.AWS_ACCESS_KEY_ID
-          AWS_SECRET_ACCESS_KEY: secrets.AWS_SECRET_ACCESS_KEY
-          CLOUDFORMATION_ROLE_ARN: secrets.CLOUDFORMATION_ROLE_ARN
-          DB_ADMIN_PASSWORD: secrets.DB_ADMIN_PASSWORD
-          DB_READ_PASSWORD: secrets.DB_READ_PASSWORD
-          CIDR_IP: secrets.CIDR_IP
-          CERTIFICATE_ARN: secrets.CERTIFICATE_ARN
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          CLOUDFORMATION_ROLE_ARN: ${{ secrets.CLOUDFORMATION_ROLE_ARN }}
+          DB_ADMIN_PASSWORD: ${{ secrets.DB_ADMIN_PASSWORD }}
+          DB_READ_PASSWORD: ${{ secrets.DB_READ_PASSWORD }}
+          CIDR_IP: ${{ secrets.CIDR_IP }}
+          CERTIFICATE_ARN: ${{ secrets.CERTIFICATE_ARN }}
           STACK_NAME: asf-stac-test
           GITHUB_BRANCH: develop
           DOMAIN_NAME: stac-test.asf.alaska.edu

From e94125b0b695cdac56a2ee291092c6b2699f2c76 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 15:03:34 -0900
Subject: [PATCH 134/144] add shell property to deploy action

---
 .github/actions/deploy-stac/action.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/actions/deploy-stac/action.yml b/.github/actions/deploy-stac/action.yml
index ef18bbeb..66fec261 100644
--- a/.github/actions/deploy-stac/action.yml
+++ b/.github/actions/deploy-stac/action.yml
@@ -37,10 +37,13 @@ runs:
       with:
         python-version: 3.9
     - name: Install Lambda dependencies
+      shell: bash
       run: make -s install-lambda-deps
     - name: Install CodeBuild dependencies
+      shell: bash
       run: python -m pip install -r requirements-run-codebuild.txt
     - name: Deploy CloudFormation stack
+      shell: bash
       run: |
         make -s deploy \
           stack_name=${{ inputs.STACK_NAME }} \
@@ -53,10 +56,12 @@ runs:
           domain_name=${{ inputs.DOMAIN_NAME }} \
           certificate_arn=${{ inputs.CERTIFICATE_ARN }}
     - name: Get CodeBuild project
+      shell: bash
       run: |
         echo "CODEBUILD_PROJECT=$(aws cloudformation describe-stacks \
                                   --stack-name ${{ inputs.STACK_NAME }} \
                                   --query 'Stacks[0].Outputs[?OutputKey==`BuildProject`].OutputValue' \
                                   --output text)" >> $GITHUB_ENV
     - name: Run CodeBuild
+      shell: bash
       run: python run_codebuild.py

From 64fa9c49c13a96b3f07da5a5b3dc8a30cbfad1b4 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 15:16:52 -0900
Subject: [PATCH 135/144] Add release workflows

---
 .github/workflows/changelog.yml | 18 ++++++++++++++++++
 .github/workflows/release.yml   | 14 ++++++++++++++
 CHANGELOG.md                    |  2 +-
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/changelog.yml
 create mode 100644 .github/workflows/release.yml

diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml
new file mode 100644
index 00000000..3f09d909
--- /dev/null
+++ b/.github/workflows/changelog.yml
@@ -0,0 +1,18 @@
+name: Changelog updated?
+
+on:
+  pull_request:
+    types:
+      - opened
+      - labeled
+      - unlabeled
+      - synchronize
+    branches:
+      - main
+      - develop
+
+jobs:
+  call-changelog-check-workflow:
+    uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.4.0
+    secrets:
+      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 00000000..b5b9dc16
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,14 @@
+name: Create Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  call-release-workflow:
+    uses: ASFHyP3/actions/.github/workflows/reusable-release.yml@v0.4.0
+    with:
+      release_prefix: ASF STAC
+    secrets:
+      USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 89f215ff..02b708bc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,6 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.0.0]
+## [0.1.0]
 ### Added
 - Initial release of STAC API endpoint backed by a PostgreSQL database

From ab47c523be6ea1dc5c2d605a515a3aac2b235ea7 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 16:19:27 -0900
Subject: [PATCH 136/144] Improve the README

---
 README.md | 78 ++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 45 insertions(+), 33 deletions(-)

diff --git a/README.md b/README.md
index c59a493e..39534005 100644
--- a/README.md
+++ b/README.md
@@ -1,32 +1,55 @@
 # asf-stac
 
-A repository containing code related to the creation and hosting of STAC catalogs by the ASF Tools team.
+Creation and hosting of STAC catalogs by the ASF Tools team.
+
+The test API is available at <https://stac-test.asf.alaska.edu>
+and the Swagger UI is available at <https://stac-test.asf.alaska.edu/api.html>.
+
+TODO: document prod URL
 
 ## Developer setup
 
-Create the environment and install developer dependencies:
+Clone the repository, create the environment, and install the developer dependencies:
 
 ```
+git clone git@github.com:ASFHyP3/asf-stac.git
+cd asf-stac
+
 conda create -n asf-stac python=3.9
 conda activate asf-stac
+
 make install
 ```
 
-## STAC API
+## Requirements for connecting to the database
 
-The test API is available at <https://stac-test.asf.alaska.edu>
-and the Swagger UI is available at <https://stac-test.asf.alaska.edu/api.html>.
+Refer to this section when manually connecting to the database or when running the API locally.
 
-TODO: document prod URL
+The database only accepts connections from within the ASF Full VPN or from clients
+with the client security group attached. See the ingress rules for the database security group in the
+[database CloudFormation template](apps/database/cloudformation.yml).
+
+The database host and database user credentials are available via the AWS Secrets Manager console
+in the AWS account where the CloudFormation stack was deployed.
+
+## Manually connecting to the database
+
+We shouldn't need to manually connect to the database during normal operations, but we can if we need to
+(e.g. for debugging purposes).
+
+Confirm you have the `psql` command installed, then run:
 
-### Running the API locally
+```
+make psql db_host=<host> db_user=<user> db_password=<password>
+```
 
-You can run the STAC API frontend locally (connected to the AWS-hosted database). This is required for accessing
-the create/update/delete endpoints (which are provided by the STAC API's Transaction extension), as these
-endpoints are disabled for the publicly accessible API.
+## Running the API locally
 
-Confirm that you're working from a machine with access to the database.
-See [Connecting to the database](#connecting-to-the-database) for exact requirements. Then run:
+You can run the STAC API frontend locally and it will automatically connect to the AWS-hosted database.
+
+The local API provides access to the Transaction extension (which provides create/update/delete endpoints),
+while the publicly available API does not. Therefore, if you need access to the Transaction endpoints, you
+must run the API locally:
 
 ```
 make run-api db_host=<host> db_admin_password=<password>
@@ -35,12 +58,18 @@ make run-api db_host=<host> db_admin_password=<password>
 You should see something like `Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit)` in the output; you can
 query the API at that URL.
 
-### Ingesting a STAC dataset
+You can confirm that the Transaction extension is enabled by opening the local API URL in a web browser
+and appending `/api.html` to open the Swagger UI. You should see various create/update/delete endpoints
+under the "Transaction Extension" heading. You should be able to successfully query these endpoints via
+the local API, but not via the publicly available API. (TODO: after removing those endpoints completely
+from the public API, update this paragraph to reflect that they will no longer appear in the Swagger UI.)
+
+## Ingesting a STAC dataset
 
-Run `python ingest_data.py -h` for usage instructions. The script only works with an API that supports the
-Transaction extension. See [Running the API locally](#running-the-api-locally).
+Run `python ingest_data.py -h` for usage instructions. You must run the ingest script against
+a locally running API (as the script requires access to the Transaction endpoints).
 
-### Upgrading the database
+## Upgrading the database
 
 The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs
 [PgSTAC](https://stac-utils.github.io/pgstac). Follow these steps to upgrade the database:
@@ -66,20 +95,3 @@ The initial AWS deployment creates a Postgres database, installs the PostGIS ext
 
 PgSTAC upgrades are handled automatically: the deployment pipeline migrates the database to the installed
 version of `pypgstac`. See <https://stac-utils.github.io/pgstac/pypgstac> for more information about migrations.
-
-### Connecting to the database
-
-We shouldn't need to manually connect to the database, but we can if we need to.
-
-The database only accepts connections from within the ASF network or from clients
-with the client security group attached. See the ingress rules for the database security group in the
-[database CloudFormation template](apps/database/cloudformation.yml).
-
-Confirm you have the `psql` command installed, then run:
-
-```
-make psql db_host=<host> db_user=<user> db_password=<password>
-```
-
-The database host and database user credentials are available via the AWS Secrets Manager console
-in the AWS account where the CloudFormation stack was deployed.

From b8030fd0e182567b87895695b6dbe91ac25a4a09 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Thu, 17 Nov 2022 16:28:25 -0900
Subject: [PATCH 137/144] include postgres download link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 39534005..71cb3a2f 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ in the AWS account where the CloudFormation stack was deployed.
 We shouldn't need to manually connect to the database during normal operations, but we can if we need to
 (e.g. for debugging purposes).
 
-Confirm you have the `psql` command installed, then run:
+Confirm that you have [PostgreSQL](https://www.postgresql.org/download/) installed, then run:
 
 ```
 make psql db_host=<host> db_user=<user> db_password=<password>

From c8592a8dfdc2a27c1c0dbf7bf36a6e36bf11169a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 11:06:04 -0900
Subject: [PATCH 138/144] Add psql to conda env

Co-authored-by: Joseph H Kennedy <me@jhkennedy.org>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 71cb3a2f..895d5da8 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ Clone the repository, create the environment, and install the developer dependen
 git clone git@github.com:ASFHyP3/asf-stac.git
 cd asf-stac
 
-conda create -n asf-stac python=3.9
+conda create -c conda-forge -n asf-stac python=3.9 postgresql
 conda activate asf-stac
 
 make install

From bdefcb3ca79f5256ce48687b590954a1cefe8e24 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 11:07:29 -0900
Subject: [PATCH 139/144] minor punctuation tweak

Co-authored-by: Joseph H Kennedy <me@jhkennedy.org>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 895d5da8..95900679 100644
--- a/README.md
+++ b/README.md
@@ -67,7 +67,7 @@ from the public API, update this paragraph to reflect that they will no longer a
 ## Ingesting a STAC dataset
 
 Run `python ingest_data.py -h` for usage instructions. You must run the ingest script against
-a locally running API (as the script requires access to the Transaction endpoints).
+a locally running API, as the script requires access to the Transaction endpoints.
 
 ## Upgrading the database
 

From 84e9da5560e560dadd28baa45fad8db1feb4d08b Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 11:08:42 -0900
Subject: [PATCH 140/144] minor clarification

Co-authored-by: Joseph H Kennedy <me@jhkennedy.org>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 95900679..cade2768 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ in the AWS account where the CloudFormation stack was deployed.
 
 ## Manually connecting to the database
 
-We shouldn't need to manually connect to the database during normal operations, but we can if we need to
+We shouldn't need to manually connect to the database during normal operations, as the API will connect automatically, but we can if we need to
 (e.g. for debugging purposes).
 
 Confirm that you have [PostgreSQL](https://www.postgresql.org/download/) installed, then run:

From cd934968226932b37a89cb8225861315495b6813 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 11:12:19 -0900
Subject: [PATCH 141/144] fix line length

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index cade2768..71ae1869 100644
--- a/README.md
+++ b/README.md
@@ -34,8 +34,8 @@ in the AWS account where the CloudFormation stack was deployed.
 
 ## Manually connecting to the database
 
-We shouldn't need to manually connect to the database during normal operations, as the API will connect automatically, but we can if we need to
-(e.g. for debugging purposes).
+We shouldn't need to manually connect to the database during normal operations,
+as the API will connect automatically, but we can if we need to (e.g. for debugging purposes).
 
 Confirm that you have [PostgreSQL](https://www.postgresql.org/download/) installed, then run:
 

From 34c440964f53752e3ea59c384a8a59daa8bfedec Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 11:32:20 -0900
Subject: [PATCH 142/144] document connecting to db from ec2 instance

---
 README.md | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/README.md b/README.md
index 71ae1869..873e3617 100644
--- a/README.md
+++ b/README.md
@@ -64,6 +64,32 @@ under the "Transaction Extension" heading. You should be able to successfully qu
 the local API, but not via the publicly available API. (TODO: after removing those endpoints completely
 from the public API, update this paragraph to reflect that they will no longer appear in the Swagger UI.)
 
+## Working with the database from an EC2 instance
+
+As explained in [Requirements for connecting to the database](#requirements-for-connecting-to-the-database),
+the database only accepts connections from particular sources. If you want to work with the database
+from an EC2 instance (e.g. because your local internet speed is insufficient for ingesting a large dataset),
+follow the steps below:
+
+1. Follow the
+   [Working from an EC2 instance](https://github.com/ASFHyP3/.github-private/wiki/Working-from-an-EC2-instance)
+   Tools Team Wiki article to create an EC2 instance in the same AWS account as the database:
+   * Instead of the default VPC, select the VPC that corresponds to the appropriate ASF STAC deployment
+     (this should be obvious from the name of the VPC in the drop-down list).
+   * After launching the instance, add an additional security group as described in the article;
+     in the list of available security groups, select the one whose name identifies it as the ASF STAC
+     database client security group.
+
+2. After SSHing into your instance and running tmux as described in the wiki article, follow
+   [Developer setup](#developer-setup) to clone and set up this repo on the EC2 instance.
+
+3. If you want to run the API on the EC2 instance (e.g. for ingesting a dataset), you can run the API as described in
+   [Running the API locally](#running-the-API-locally). After the API is running, you can create a new tmux
+   window to accomplish more work, such as running the ingest script.
+
+4. If you're ingesting a dataset that is stored in S3, you can use the AWS CLI to download the S3 object
+   to the EC2 instance before running the ingest script.
+
 ## Ingesting a STAC dataset
 
 Run `python ingest_data.py -h` for usage instructions. You must run the ingest script against

From a5f9e711511f7b8b9a8cf0b084f175ca982f742a Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 11:41:24 -0900
Subject: [PATCH 143/144] move a section

---
 README.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 873e3617..dec4502f 100644
--- a/README.md
+++ b/README.md
@@ -64,6 +64,11 @@ under the "Transaction Extension" heading. You should be able to successfully qu
 the local API, but not via the publicly available API. (TODO: after removing those endpoints completely
 from the public API, update this paragraph to reflect that they will no longer appear in the Swagger UI.)
 
+## Ingesting a STAC dataset
+
+Run `python ingest_data.py -h` for usage instructions. You must run the ingest script against
+a locally running API, as the script requires access to the Transaction endpoints.
+
 ## Working with the database from an EC2 instance
 
 As explained in [Requirements for connecting to the database](#requirements-for-connecting-to-the-database),
@@ -90,11 +95,6 @@ follow the steps below:
 4. If you're ingesting a dataset that is stored in S3, you can use the AWS CLI to download the S3 object
    to the EC2 instance before running the ingest script.
 
-## Ingesting a STAC dataset
-
-Run `python ingest_data.py -h` for usage instructions. You must run the ingest script against
-a locally running API, as the script requires access to the Transaction endpoints.
-
 ## Upgrading the database
 
 The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs

From ee6c24660a57575eb72e3d9bc2e5a81e08132fa4 Mon Sep 17 00:00:00 2001
From: Jake Herrmann <jtherrmann@alaska.edu>
Date: Fri, 18 Nov 2022 14:59:04 -0900
Subject: [PATCH 144/144] Remove README section

---
 README.md | 26 --------------------------
 1 file changed, 26 deletions(-)

diff --git a/README.md b/README.md
index dec4502f..71ae1869 100644
--- a/README.md
+++ b/README.md
@@ -69,32 +69,6 @@ from the public API, update this paragraph to reflect that they will no longer a
 Run `python ingest_data.py -h` for usage instructions. You must run the ingest script against
 a locally running API, as the script requires access to the Transaction endpoints.
 
-## Working with the database from an EC2 instance
-
-As explained in [Requirements for connecting to the database](#requirements-for-connecting-to-the-database),
-the database only accepts connections from particular sources. If you want to work with the database
-from an EC2 instance (e.g. because your local internet speed is insufficient for ingesting a large dataset),
-follow the steps below:
-
-1. Follow the
-   [Working from an EC2 instance](https://github.com/ASFHyP3/.github-private/wiki/Working-from-an-EC2-instance)
-   Tools Team Wiki article to create an EC2 instance in the same AWS account as the database:
-   * Instead of the default VPC, select the VPC that corresponds to the appropriate ASF STAC deployment
-     (this should be obvious from the name of the VPC in the drop-down list).
-   * After launching the instance, add an additional security group as described in the article;
-     in the list of available security groups, select the one whose name identifies it as the ASF STAC
-     database client security group.
-
-2. After SSHing into your instance and running tmux as described in the wiki article, follow
-   [Developer setup](#developer-setup) to clone and set up this repo on the EC2 instance.
-
-3. If you want to run the API on the EC2 instance (e.g. for ingesting a dataset), you can run the API as described in
-   [Running the API locally](#running-the-API-locally). After the API is running, you can create a new tmux
-   window to accomplish more work, such as running the ingest script.
-
-4. If you're ingesting a dataset that is stored in S3, you can use the AWS CLI to download the S3 object
-   to the EC2 instance before running the ingest script.
-
 ## Upgrading the database
 
 The initial AWS deployment creates a Postgres database, installs the PostGIS extension, and then installs