Merge pull request #94 from ASFHyP3/develop

Release 0.5.2

.github/workflows/tag-version.yml

@@ -7,7 +7,6 @@ on:
 
 jobs:
   call-bump-version-workflow:
-    needs: deploy
     uses: ASFHyP3/actions/.github/workflows/[email protected]
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.2]
+
+### Changed
+- Sentinel-2 products are now disqualified from processing if they do not have enough data coverage.
+- Sentinel-2 products are now disqualified from processing if the secondary scene's relative orbit does not match that of the reference scene.
+- Switched from Dataspace's Sentinel-2 STAC API to Element84's.
+
 ## [0.5.1]
 
 ### Fixed

its_live_monitoring/src/landsat.py

@@ -11,15 +11,16 @@
 import pystac
 import pystac_client
 
-from constants import MAX_CLOUD_COVER_PERCENT, MAX_PAIR_SEPARATION_IN_DAYS
-
 
 LANDSAT_CATALOG_API = 'https://landsatlook.usgs.gov/stac-server'
 LANDSAT_CATALOG = pystac_client.Client.open(LANDSAT_CATALOG_API)
 LANDSAT_COLLECTION_NAME = 'landsat-c2l1'
 LANDSAT_COLLECTION = LANDSAT_CATALOG.get_collection(LANDSAT_COLLECTION_NAME)
 LANDSAT_TILES_TO_PROCESS = json.loads((Path(__file__).parent / 'landsat_tiles_to_process.json').read_text())
 
+LANDSAT_MAX_PAIR_SEPARATION_IN_DAYS = 544
+LANDSAT_MAX_CLOUD_COVER_PERCENT = 60
+
 log = logging.getLogger('its_live_monitoring')
 log.setLevel(os.environ.get('LOGGING_LEVEL', 'INFO'))
 
@@ -35,7 +36,7 @@ def get_landsat_stac_item(scene: str) -> pystac.Item:  # noqa: D103
 
 
 def qualifies_for_landsat_processing(
-    item: pystac.item.Item, max_cloud_cover: int = MAX_CLOUD_COVER_PERCENT, log_level: int = logging.DEBUG
+    item: pystac.item.Item, *, max_cloud_cover: int = LANDSAT_MAX_CLOUD_COVER_PERCENT, log_level: int = logging.DEBUG
 ) -> bool:
     """Determines whether a scene is a valid Landsat product for processing.
 
@@ -77,8 +78,9 @@ def qualifies_for_landsat_processing(
 
 def get_landsat_pairs_for_reference_scene(
     reference: pystac.item.Item,
-    max_pair_separation: timedelta = timedelta(days=MAX_PAIR_SEPARATION_IN_DAYS),
-    max_cloud_cover: int = MAX_CLOUD_COVER_PERCENT,
+    *,
+    max_pair_separation: timedelta = timedelta(days=LANDSAT_MAX_PAIR_SEPARATION_IN_DAYS),
+    max_cloud_cover: int = LANDSAT_MAX_CLOUD_COVER_PERCENT,
 ) -> gpd.GeoDataFrame:
     """Generate potential ITS_LIVE velocity pairs for a given Landsat scene.
 
@@ -102,7 +104,10 @@ def get_landsat_pairs_for_reference_scene(
     )
 
     items = [
-        item for page in results.pages() for item in page if qualifies_for_landsat_processing(item, max_cloud_cover)
+        item
+        for page in results.pages()
+        for item in page
+        if qualifies_for_landsat_processing(item, max_cloud_cover=max_cloud_cover)
     ]
 
     log.debug(f'Found {len(items)} secondary scenes for {reference.id}')

its_live_monitoring/src/main.py

@@ -5,13 +5,11 @@
 import logging
 import os
 import sys
-from datetime import timedelta
 
 import geopandas as gpd
 import hyp3_sdk as sdk
 import pandas as pd
 
-from constants import MAX_CLOUD_COVER_PERCENT, MAX_PAIR_SEPARATION_IN_DAYS
 from landsat import (
     get_landsat_pairs_for_reference_scene,
     get_landsat_stac_item,
@@ -88,35 +86,30 @@ def submit_pairs_for_processing(pairs: gpd.GeoDataFrame) -> sdk.Batch:  # noqa:
 
 def process_scene(
     scene: str,
-    max_pair_separation: timedelta = timedelta(days=MAX_PAIR_SEPARATION_IN_DAYS),
-    max_cloud_cover: int = MAX_CLOUD_COVER_PERCENT,
     submit: bool = True,
 ) -> sdk.Batch:
     """Trigger Landsat processing for a scene.
 
     Args:
         scene: Reference Landsat scene name to build pairs for.
-        max_pair_separation: How many days back from a reference scene's acquisition date to search for secondary
-         scenes.
-        max_cloud_cover: The maximum percent a Landsat scene can be covered by clouds.
         submit: Submit pairs to HyP3 for processing.
 
     Returns:
         Jobs submitted to HyP3 for processing.
     """
     pairs = None
     if scene.startswith('S2'):
-        reference = get_sentinel2_stac_item(f'{scene}.SAFE')
-        if qualifies_for_sentinel2_processing(reference, max_cloud_cover, logging.INFO):
+        reference = get_sentinel2_stac_item(scene)
+        if qualifies_for_sentinel2_processing(reference, log_level=logging.INFO):
             # hyp3-its-live will pull scenes from Google Cloud; ensure the new scene is there before processing
             # Note: Time between attempts is controlled by they SQS VisibilityTimout
             _ = raise_for_missing_in_google_cloud(scene)
-            pairs = get_sentinel2_pairs_for_reference_scene(reference, max_pair_separation, max_cloud_cover)
+            pairs = get_sentinel2_pairs_for_reference_scene(reference)
 
     else:
         reference = get_landsat_stac_item(scene)
-        if qualifies_for_landsat_processing(reference, max_cloud_cover, logging.INFO):
-            pairs = get_landsat_pairs_for_reference_scene(reference, max_pair_separation, max_cloud_cover)
+        if qualifies_for_landsat_processing(reference, log_level=logging.INFO):
+            pairs = get_landsat_pairs_for_reference_scene(reference)
 
     if pairs is None:
         return sdk.Batch()
@@ -169,18 +162,6 @@ def main() -> None:
     """Command Line wrapper around `process_scene`."""
     parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter)
     parser.add_argument('reference', help='Reference Landsat scene name to build pairs for')
-    parser.add_argument(
-        '--max-pair-separation',
-        type=int,
-        default=MAX_PAIR_SEPARATION_IN_DAYS,
-        help="How many days back from a reference scene's acquisition date to search for secondary scenes",
-    )
-    parser.add_argument(
-        '--max-cloud-cover',
-        type=int,
-        default=MAX_CLOUD_COVER_PERCENT,
-        help='The maximum percent a Landsat scene can be covered by clouds',
-    )
     parser.add_argument('--submit', action='store_true', help='Submit pairs to HyP3 for processing')
     parser.add_argument('-v', '--verbose', action='store_true', help='Turn on verbose logging')
     args = parser.parse_args()
@@ -190,7 +171,7 @@ def main() -> None:
         log.setLevel(logging.DEBUG)
 
     log.debug(' '.join(sys.argv))
-    _ = process_scene(args.reference, timedelta(days=args.max_pair_separation), args.max_cloud_cover, args.submit)
+    _ = process_scene(args.reference, submit=args.submit)
 
 
 if __name__ == '__main__':