Update test scripts

AlexeyPechnikov · Feb 25, 2024 · 12e1afb · 12e1afb
1 parent 1961f07
commit 12e1afb
Show file tree

Hide file tree

Showing 2 changed files with 99 additions and 37 deletions.
diff --git a/tests/imperial_valley_2015.py b/tests/imperial_valley_2015.py
@@ -93,8 +93,8 @@
 !{sys.executable} --version
 
 if 'google.colab' in sys.modules:
-    #!{sys.executable} -m pip install -q git+https://github.com/mobigroup/gmtsar.git@pygmtsar2#subdirectory=pygmtsar
-    !{sys.executable} -m pip install -q pygmtsar
+    !{sys.executable} -m pip install -q git+https://github.com/mobigroup/gmtsar.git@pygmtsar2#subdirectory=pygmtsar
+    #!{sys.executable} -m pip install -q pygmtsar
 from pygmtsar import __version__
 __version__
 
@@ -132,7 +132,7 @@ def mpl_settings(settings):
 plt.rcParams['ytick.labelsize'] = 12
 # %matplotlib inline
 
-from pygmtsar import S1, Stack, tqdm_dask, NCubeVTK, ASF
+from pygmtsar import S1, Stack, tqdm_dask, NCubeVTK, ASF, AWS, ESA, GMT
 
 """## Define Sentinel-1 SLC Scenes and Processing Parameters"""
 
@@ -150,24 +150,56 @@ def mpl_settings(settings):
 BASEDAYS     = 100
 BASEMETERS   = 150
 
+# define DEM filename inside data directory
+DEM = f'{DATADIR}/dem.nc'
+
 """## Download and Unpack Datasets
 
-## Enter Your ASF (Earthdata) User and Password
+## Enter Your ASF and ESA User and Password
 
-If the data directory is empty or doesn't exist, you'll need to download Sentinel-1 scenes from the Alaska Satellite Facility (ASF) datastore. Use your Earthdata Login credentials. If you don't have an Earthdata Login, you can create one at https://urs.earthdata.nasa.gov//users/new
+If the data directory is empty or doesn't exist, you'll need to download Sentinel-1 scenes from the Alaska Satellite Facility (ASF) datastore. Use your Earthdata Login credentials. If you don't have an Earthdata Login, you can create one at https://urs.earthdata.nasa.gov//users/new Also, register your ESA Copernicus datastore account at https://dataspace.copernicus.eu/
 
 You can also use pre-existing SLC scenes stored on your Google Drive, or you can copy them using a direct public link from iCloud Drive.
 
-The credentials below are available at the time the notebook is validated. Special symbols, like underscores, are required in your ASF password.
+The credentials below are available at the time the notebook is validated.
 """
 
 # Set these variables to None and you will be prompted to enter your username and password below.
-username = 'GoogleColab2023'
-password = 'GoogleColab_2023'
+asf_username = 'GoogleColab2023'
+asf_password = 'GoogleColab_2023'
+
+esa_username = '[email protected]'
+esa_password = 'cnjwdchuwe&e9d0We9'
 
-# download required polarization and subswaths only
-asf = ASF(username, password)
-asf.download(DATADIR, SCENES, SUBSWATH, POLARIZATION)
+# Set these variables to None and you will be prompted to enter your username and password below.
+asf = ASF(asf_username, asf_password)
+# Optimized scene downloading from ASF - only the required subswaths and polarizations.
+print(asf.download_scenes(DATADIR, SCENES, SUBSWATH))
+# There are two ways to download orbits; you can use any one or both together.
+try:
+    # RESORB orbit downloading from ASF has recently failed.
+    print(asf.download_orbits(DATADIR))
+except Exception as e:
+    print (e)
+    # Download missed orbits in case ASF orbit downloading fails.
+    esa = ESA(esa_username, esa_password)
+    print (esa.download_orbits(DATADIR))
+
+# define AOI as the whole scenes area
+AOI = S1.scan_slc(DATADIR)
+
+# previously, PyGMTSAR internally applied 0.1° buffer
+try:
+    # download SRTM DEM from GMT servers
+    # note: downloading often fails recently
+    GMT().download_dem(AOI, filename=DEM)
+except Exception as e:
+    print (e)
+
+# if DEM missed, download Copernicus DEM from open AWS datastore
+# get complete 1°x1° tiles covering the AOI, crop them later using AOI
+AWS().download_dem(AOI, filename=DEM)
+# don't worry about messages 'ERROR 3: /vsipythonfilelike/ ... : I/O error'
 
 """## Run Local Dask Cluster
 
@@ -193,13 +225,13 @@ def mpl_settings(settings):
 
 sbas.plot_scenes()
 
-"""### Download SRTM DEM
+"""### Load DEM
 
-The function below downloads SRTM1 or SRTM3 DEM and converts heights to ellipsoidal model using EGM96 grid.
-Besides, for faster processing we can use pre-defined DEM file as explained above. Select product=SRTM1 for 30m resolution and product=SRTM3 for 90m resolution SRTM DEM.
+The function below loads DEM from file or Xarray variable and converts heights to ellipsoidal model using EGM96 grid.
 """
 
-sbas.download_dem()
+# define the area of interest (AOI) to speedup the processing
+sbas.load_dem(DEM, AOI)
 
 sbas.plot_scenes()
 plt.savefig('Estimated Scene Locations.jpg')
@@ -217,7 +249,8 @@ def mpl_settings(settings):
 baseline_pairs = sbas.baseline_pairs(days=BASEDAYS, meters=BASEMETERS)
 baseline_pairs
 
-sbas.plot_baseline(baseline_pairs)
+with mpl_settings({'figure.dpi': 150}):
+    sbas.plot_baseline(baseline_pairs)
 plt.savefig('Baseline.jpg')
 
 """## Geocoding"""

diff --git a/tests/pico_do_fogo_volcano_eruption_2014.py b/tests/pico_do_fogo_volcano_eruption_2014.py
@@ -93,8 +93,8 @@
 !{sys.executable} --version
 
 if 'google.colab' in sys.modules:
-    #!{sys.executable} -m pip install -Uq git+https://github.com/mobigroup/gmtsar.git@pygmtsar2#subdirectory=pygmtsar
-    !{sys.executable} -m pip install -q pygmtsar
+    !{sys.executable} -m pip install -Uq git+https://github.com/mobigroup/gmtsar.git@pygmtsar2#subdirectory=pygmtsar
+    #!{sys.executable} -m pip install -q pygmtsar
 from pygmtsar import __version__
 __version__
 
@@ -138,7 +138,7 @@ def mpl_settings(settings):
 pd.set_option('display.width', None)
 pd.set_option('display.max_colwidth', 100)
 
-from pygmtsar import S1, Stack, tqdm_dask, NCubeVTK, ASF
+from pygmtsar import S1, Stack, tqdm_dask, NCubeVTK, ASF, AWS, ESA, GMT
 
 """## Define Processing Parameters"""
 
@@ -149,6 +149,10 @@ def mpl_settings(settings):
 WORKDIR      = 'raw_fogo'
 DATADIR      = 'data_fogo'
 
+# define DEM and landmask filenames inside data directory
+DEM = f'{DATADIR}/dem.nc'
+LANDMASK = f'{DATADIR}/landmask.nc'
+
 geojson = '''
 {
   "type": "Feature",
@@ -163,22 +167,51 @@ def mpl_settings(settings):
 
 """## Download and Unpack Datasets
 
-## Enter Your ASF (Earthdata) User and Password
+## Enter Your ASF and ESA User and Password
 
-If the data directory is empty or doesn't exist, you'll need to download Sentinel-1 scenes from the Alaska Satellite Facility (ASF) datastore. Use your Earthdata Login credentials. If you don't have an Earthdata Login, you can create one at https://urs.earthdata.nasa.gov//users/new
+If the data directory is empty or doesn't exist, you'll need to download Sentinel-1 scenes from the Alaska Satellite Facility (ASF) datastore. Use your Earthdata Login credentials. If you don't have an Earthdata Login, you can create one at https://urs.earthdata.nasa.gov//users/new Also, register your ESA Copernicus datastore account at https://dataspace.copernicus.eu/
 
 You can also use pre-existing SLC scenes stored on your Google Drive, or you can copy them using a direct public link from iCloud Drive.
 
-The credentials below are available at the time the notebook is validated. Special symbols, like underscores, are required in your ASF password.
+The credentials below are available at the time the notebook is validated.
 """
 
 # Set these variables to None and you will be prompted to enter your username and password below.
-username = 'GoogleColab2023'
-password = 'GoogleColab_2023'
+asf_username = 'GoogleColab2023'
+asf_password = 'GoogleColab_2023'
+
+esa_username = '[email protected]'
+esa_password = 'cnjwdchuwe&e9d0We9'
 
-# download required polarization and subswaths only
-asf = ASF(username, password)
-asf.download(DATADIR, SCENES, SUBSWATH)
+# Set these variables to None and you will be prompted to enter your username and password below.
+asf = ASF(asf_username, asf_password)
+# Optimized scene downloading from ASF - only the required subswaths and polarizations.
+print(asf.download_scenes(DATADIR, SCENES, SUBSWATH))
+# There are two ways to download orbits; you can use any one or both together.
+try:
+    # RESORB orbit downloading from ASF has recently failed.
+    print(asf.download_orbits(DATADIR))
+except Exception as e:
+    print (e)
+    # Download missed orbits in case ASF orbit downloading fails.
+    esa = ESA(esa_username, esa_password)
+    print (esa.download_orbits(DATADIR))
+
+# previously, PyGMTSAR internally applied 0.1° buffer
+try:
+    # download SRTM DEM from GMT servers
+    # note: downloading often fails recently
+    GMT().download_dem(AOI, filename=DEM)
+except Exception as e:
+    print (e)
+
+# if DEM missed, download Copernicus DEM from open AWS datastore
+# get complete 1°x1° tiles covering the AOI, crop them later using AOI
+AWS().download_dem(AOI, filename=DEM)
+# don't worry about messages 'ERROR 3: /vsipythonfilelike/ ... : I/O error'
+
+# download SRTM DEM from GMT servers or make locally when gmt-gshhg installed
+GMT().download_landmask(AOI, filename=LANDMASK)
 
 """## Run Local Dask Cluster
 
@@ -213,23 +246,19 @@ def mpl_settings(settings):
 
 sbas.plot_scenes(AOI=AOI, aspect='equal')
 
-"""### Download SRTM DEM
-
-The function below downloads SRTM1 or SRTM3 DEM and converts heights to ellipsoidal model using EGM96 grid.
-Besides, for faster processing we can use pre-defined DEM file as explained above.
-
-SRTM1 product is 30m resolution DEM and SRTM3 is 90m. SRTM1 is much bigger (~10 times) and is usable for small areas. Mainly 90m SRTM3 is the right choice. Use parameter resolution_meters (60 meters by default) to interpolate the DEM to required resolution for the future processing and output.
+"""### Load DEM
 
-The DEM grid is NetCDF file.
+The function below loads DEM from file or Xarray variable and converts heights to ellipsoidal model using EGM96 grid.
 """
 
-sbas.download_dem(AOI)
+# define the area of interest (AOI) to speedup the processing
+sbas.load_dem(DEM, AOI)
 
 sbas.plot_scenes(AOI=AOI, aspect='equal')
 
-"""## Download Landmask"""
+"""## Load Landmask"""
 
-sbas.download_landmask()
+sbas.load_landmask(LANDMASK)
 
 sbas.plot_scenes(AOI=AOI, dem=sbas.get_dem().where(sbas.get_landmask()), caption='Sentinel1 Landmasked Frame on DEM', aspect='equal')
 plt.savefig('Sentinel1 Landmasked Frame on DEM.jpg')