From 54c17a69d777cff1d39ff1bbe4f6a80a2c9c920b Mon Sep 17 00:00:00 2001 From: tsutterley Date: Fri, 21 Aug 2020 14:52:34 -0700 Subject: [PATCH] add Arctic AOTIM-5-2018 matlab-python test add download utilties for CATS and Arctic models update documentation for setting up model directories update documentation for new and updated programs --- .github/workflows/python-package.yml | 1 + .github/workflows/python-request.yml | 1 + Plot Antarctic Tide Range.ipynb | 2 +- doc/source/conf.py | 2 +- doc/source/getting_started/Getting-Started.md | 45 ++++++ doc/source/index.rst | 3 + doc/source/user_guide/arcticdata_tides.md | 21 +++ doc/source/user_guide/usap_cats_tides.md | 17 +++ doc/source/user_guide/utilities.rst | 9 ++ pyTMD/convert_ll_xy.py | 2 +- pyTMD/infer_minor_corrections.py | 2 +- pyTMD/time.py | 1 + pyTMD/utilities.py | 22 +++ scripts/arcticdata_tides.py | 119 ++++++++++++++++ scripts/usap_cats_tides.py | 106 +++++++++++++++ setup.py | 2 +- test/test_download_and_read.py | 128 +++++++++++++++++- 17 files changed, 473 insertions(+), 10 deletions(-) create mode 100644 doc/source/getting_started/Getting-Started.md create mode 100644 doc/source/user_guide/arcticdata_tides.md create mode 100644 doc/source/user_guide/usap_cats_tides.md create mode 100644 scripts/arcticdata_tides.py create mode 100644 scripts/usap_cats_tides.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5dfce732..4c5ee19b 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -27,6 +27,7 @@ jobs: - name: Install dependencies for Linux if: matrix.os == 'ubuntu-20.04' run: | + sudo apt-get update sudo apt-get install libproj-dev proj-data proj-bin libgeos-dev octave sudo apt-get install libhdf5-dev libnetcdf-dev pip install --upgrade pip diff --git a/.github/workflows/python-request.yml b/.github/workflows/python-request.yml index f9603c84..a5f7c506 100644 --- a/.github/workflows/python-request.yml +++ b/.github/workflows/python-request.yml @@ -24,6 +24,7 @@ jobs: - name: Install dependencies for Linux if: matrix.os == 'ubuntu-20.04' run: | + sudo apt-get update sudo apt-get install libproj-dev proj-data proj-bin libgeos-dev octave sudo apt-get install libhdf5-dev libnetcdf-dev pip install --upgrade pip diff --git a/Plot Antarctic Tide Range.ipynb b/Plot Antarctic Tide Range.ipynb index 00ebb03d..877e7adb 100644 --- a/Plot Antarctic Tide Range.ipynb +++ b/Plot Antarctic Tide Range.ipynb @@ -285,7 +285,7 @@ " #-- list of minor constituents\n", " minor = ['2q1','sigma1','rho1','m1','m1','chi1','pi1','phi1','theta1',\n", " 'j1','oo1','2n2','mu2','nu2','lambda2','l2','l2','t2']\n", - " #-- only add minor consituents that are not on the list of major values\n", + " #-- only add minor constituents that are not on the list of major values\n", " minor_flag = [m not in constituents for m in minor]\n", " #-- estimate minor constituents\n", " zmin = np.zeros((npts,18))\n", diff --git a/doc/source/conf.py b/doc/source/conf.py index 1da1697d..6478e80a 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -22,7 +22,7 @@ author = 'Tyler C. Sutterley' # The full version, including alpha/beta/rc tags -release = '1.0.2.7' +release = '1.0.2.8' # -- General configuration --------------------------------------------------- diff --git a/doc/source/getting_started/Getting-Started.md b/doc/source/getting_started/Getting-Started.md new file mode 100644 index 00000000..2ba1c94c --- /dev/null +++ b/doc/source/getting_started/Getting-Started.md @@ -0,0 +1,45 @@ +Getting Started +=============== + +#### Tide Model Formats +OTIS and ATLAS formatted data use a single binary file to store all the constituents for either heights (`z`) or transports (`u`, `v`). +Arctic Ocean models can be downloaded from the NSF ArcticData server using the [`arcticdata_tides.py`](https://github.com/tsutterley/pyTMD/blob/master/scripts/arcticdata_tides.py) program. +CATS2008 can be downloaded from the US Antarctic Program (USAP) using the [`usap_cats_tides.py`](https://github.com/tsutterley/pyTMD/blob/master/scripts/usap_cats_tides.py) program. +ATLAS netCDF formatted data use netCDF4 files for each constituent and variable type (`z`, `u`, `v`). +GOT formatted data use ascii files for each height constituent (`z`). +FES formatted data use either ascii (1999, 2004) or netCDF4 (2012, 2014) files for each constituent and variable type (`z`, `u`, `v`). +The FES models can be downloaded using the [`aviso_fes_tides.py`](https://github.com/tsutterley/pyTMD/blob/master/scripts/aviso_fes_tides.py) program for users registered with AVISO. + +#### Directories +pyTMD uses a tree structure for storing the tidal constituent data. +This structure was chosen based on the different formats of each tide model. + +- Circum-Antarctic Tidal Simulations + * CATS0201: `/cats0201_tmd/` + * [CATS2008](https://www.usap-dc.org/view/dataset/601235): `/CATS2008/` + * CATS2008_load: `/CATS2008a_SPOTL_Load/` + +- Arctic Ocean Tidal Simulations + * [AODTM-5](https://arcticdata.io/catalog/view/doi:10.18739/A2901ZG3N): `/aodtm5_tmd/` + * [AOTIM-5](https://arcticdata.io/catalog/view/doi:10.18739/A2S17SS80): `/aotim5_tmd/` + * [AOTIM-5-2018](https://arcticdata.io/catalog/view/doi:10.18739/A21R6N14K): `/Arc5km2018/` + +- TOPEX/POSEIDON global tide models + * [TPXO9-atlas](https://www.tpxo.net/tpxo-products-and-registration): `/TPXO9_atlas/` + * [TPXO9-atlas-v2](https://www.tpxo.net/tpxo-products-and-registration): `/TPXO9_atlas_v2/` + * [TPXO9.1](https://www.tpxo.net/tpxo-products-and-registration): `/TPXO9.1/DATA/` + * [TPXO8-atlas](https://www.tpxo.net/tpxo-products-and-registration): `/tpxo8_atlas/` + * TPXO7.2: `/TPXO7.2_tmd/` + * TPXO7.2_load: `/TPXO7.2_load/` + +- Global Ocean Tide models + * GOT4.7: `/GOT4.7/grids_oceantide/` + * GOT4.7_load: `/GOT4.7/grids_loadtide/` + * GOT4.8: `/got4.8/grids_oceantide/` + * GOT4.8_load: `/got4.8/grids_loadtide/` + * GOT4.10: `/GOT4.10c/grids_oceantide/` + * GOT4.10_load: `/GOT4.10c/grids_loadtide/` + +- Finite Element Solution tide models + * [FES2014](https://www.aviso.altimetry.fr/data/products/auxiliary-products/global-tide-fes.html): `/fes2014/ocean_tide/` + * [FES2014_load](https://www.aviso.altimetry.fr/data/products/auxiliary-products/global-tide-fes.html): `/fes2014/load_tide/` diff --git a/doc/source/index.rst b/doc/source/index.rst index 33a61694..6e530a30 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,6 +26,7 @@ calculating radial pole tide displacements :caption: Getting Started: getting_started/Install.md + getting_started/Getting-Started.md getting_started/Resources.md getting_started/Citations.md @@ -33,6 +34,7 @@ calculating radial pole tide displacements :maxdepth: 1 :caption: User Guide: + user_guide/arcticdata_tides.md user_guide/aviso_fes_tides.md user_guide/bilinear_interp.md user_guide/calc_astrol_longitudes.md @@ -70,4 +72,5 @@ calculating radial pole tide displacements user_guide/reduce_OTIS_files.md user_guide/tidal_ellipse.md user_guide/time.rst + user_guide/usap_cats_tides.md user_guide/utilities.rst diff --git a/doc/source/user_guide/arcticdata_tides.md b/doc/source/user_guide/arcticdata_tides.md new file mode 100644 index 00000000..cc29bcce --- /dev/null +++ b/doc/source/user_guide/arcticdata_tides.md @@ -0,0 +1,21 @@ +arcticdata_tides.py +=================== + + - Download Arctic Ocean Tide Models from the [NSF ArcticData](https://arcticdata.io) archive + - [AODTM-5](https://arcticdata.io/catalog/view/doi:10.18739/A2901ZG3N) + - [AOTIM-5](https://arcticdata.io/catalog/view/doi:10.18739/A2S17SS80) + - [AOTIM-5-2018](https://arcticdata.io/catalog/view/doi:10.18739/A21R6N14K) + +#### Calling Sequence +```bash +python arcticdata_tides.py --directory= --tide=AOTIM-5-2018 +``` +[Source code](https://github.com/tsutterley/pyTMD/blob/master/scripts/arcticdata_tides.py) + +#### Command Line Options + - `-D X`, `--directory=X`: Working Data Directory + - `--tide=X`: Arctic Ocean tide model to download + * AODTM-5 + * AOTIM-5 + * AOTIM-5-2018 + - `-M X`, `--mode=X`: Permission mode of files downloaded diff --git a/doc/source/user_guide/usap_cats_tides.md b/doc/source/user_guide/usap_cats_tides.md new file mode 100644 index 00000000..7934217f --- /dev/null +++ b/doc/source/user_guide/usap_cats_tides.md @@ -0,0 +1,17 @@ +usap_cats_tides.py +================== + + - Download Circum-Antarctic Tidal Simulations from the [US Antarctic Program](https://www.usap-dc.org) + - [CATS2008](https://www.usap-dc.org/view/dataset/601235) + +#### Calling Sequence +```bash +python usap_cats_tides.py --directory= --tide=CATS2008 +``` +[Source code](https://github.com/tsutterley/pyTMD/blob/master/scripts/usap_cats_tides.py) + +#### Command Line Options + - `-D X`, `--directory=X`: Working Data Directory + - `--tide=X`: Circum-Antarctic Tidal Simulation to download + * CATS2008 + - `-M X`, `--mode=X`: Permission mode of files downloaded diff --git a/doc/source/user_guide/utilities.rst b/doc/source/user_guide/utilities.rst index af92c40b..d7c01cff 100644 --- a/doc/source/user_guide/utilities.rst +++ b/doc/source/user_guide/utilities.rst @@ -105,6 +105,15 @@ General Methods `mode`: permissions mode of output local file +.. method:: pyTMD.utilities.check_connection(HOST) + + Check internet connection + + Arguments: + + `HOST`: remote http host + + .. method:: pyTMD.utilities.from_http(HOST,timeout=None,local=None,hash='',chunk=16384,verbose=False,mode=0o775) Download a file from a http host diff --git a/pyTMD/convert_ll_xy.py b/pyTMD/convert_ll_xy.py index 73ee53ad..3f94cc12 100644 --- a/pyTMD/convert_ll_xy.py +++ b/pyTMD/convert_ll_xy.py @@ -108,7 +108,7 @@ def xy_ll_EPSG3976(i1,i2,BF): #-- return the output variables return (o1,o2) -#-- wrapper function for models in PSNorth projection +#-- wrapper function for models in (idealized) PSNorth projection def xy_ll_PSNorth(i1,i2,BF): # #-- projections for converting from latitude/longitude # proj1 = pyproj.Proj("+init=EPSG:{0:d}".format(4326)) diff --git a/pyTMD/infer_minor_corrections.py b/pyTMD/infer_minor_corrections.py index acf84c77..6f64a590 100755 --- a/pyTMD/infer_minor_corrections.py +++ b/pyTMD/infer_minor_corrections.py @@ -94,7 +94,7 @@ def infer_minor_corrections(time,zmajor,constituents,DELTAT=0.0,CORRECTIONS=''): #-- list of minor constituents minor = ['2q1','sigma1','rho1','m1','m1','chi1','pi1','phi1','theta1','j1', 'oo1','2n2','mu2','nu2','lambda2','l2','l2','t2'] - #-- only add minor consituents that are not on the list of major values + #-- only add minor constituents that are not on the list of major values minor_indices = [i for i,m in enumerate(minor) if m not in constituents] #-- relationship between major and minor constituent amplitude and phase diff --git a/pyTMD/time.py b/pyTMD/time.py index d6fc409d..dfba287c 100644 --- a/pyTMD/time.py +++ b/pyTMD/time.py @@ -6,6 +6,7 @@ PYTHON DEPENDENCIES: numpy: Scientific Computing Tools For Python (https://numpy.org) + lxml: processing XML and HTML in Python (https://pypi.python.org/pypi/lxml) PROGRAM DEPENDENCIES: convert_julian.py: returns the calendar date and time given a Julian date diff --git a/pyTMD/utilities.py b/pyTMD/utilities.py index ea88d304..45edf886 100644 --- a/pyTMD/utilities.py +++ b/pyTMD/utilities.py @@ -3,6 +3,9 @@ Written by Tyler Sutterley (08/2020) Download and management utilities for syncing time and auxiliary files +PYTHON DEPENDENCIES: + lxml: processing XML and HTML in Python (https://pypi.python.org/pypi/lxml) + UPDATE HISTORY: Updated 08/2020: add GSFC CDDIS opener, login and download functions Written 08/2020 @@ -24,9 +27,11 @@ import lxml.etree import calendar,time if sys.version_info[0] == 2: + from urllib import quote_plus from cookielib import CookieJar import urllib2 else: + from urllib.parse import quote_plus from http.cookiejar import CookieJar import urllib.request as urllib2 @@ -228,6 +233,23 @@ def from_ftp(HOST,timeout=None,local=None,hash='',chunk=16384, remote_buffer.seek(0) return remote_buffer +#-- PURPOSE: check internet connection +def check_connection(HOST): + """ + Check internet connection + + Arguments + --------- + HOST: remote http host + """ + #-- attempt to connect to https host + try: + urllib2.urlopen(HOST,timeout=20,context=ssl.SSLContext()) + except urllib2.URLError: + raise RuntimeError('Check internet connection') + else: + return True + #-- PURPOSE: download a file from a http host def from_http(HOST,timeout=None,local=None,hash='',chunk=16384, verbose=False,mode=0o775): diff --git a/scripts/arcticdata_tides.py b/scripts/arcticdata_tides.py new file mode 100644 index 00000000..611a7610 --- /dev/null +++ b/scripts/arcticdata_tides.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +u""" +arcticdata_tides.py +Written by Tyler Sutterley (07/2020) +Download Arctic Ocean Tide Models from the NSF ArcticData archive +AODTM-5: https://arcticdata.io/catalog/view/doi:10.18739/A2901ZG3N +AOTIM-5: https://arcticdata.io/catalog/view/doi:10.18739/A2S17SS80 +AOTIM-5-2018: https://arcticdata.io/catalog/view/doi:10.18739/A21R6N14K + +CALLING SEQUENCE: + python arcticdata_tides.py --tide=AOTIM-5-2018 + +COMMAND LINE OPTIONS: + --help: list the command line options + --directory=X: working data directory + --tide=X: tide model to download + AODTM-5 + AOTIM-5 + AOTIM-5-2018 + -M X, --mode=X: Local permissions mode of the files downloaded + +PYTHON DEPENDENCIES: + future: Compatibility layer between Python 2 and Python 3 + https://python-future.org/ + +PROGRAM DEPENDENCIES: + utilities: download and management utilities for syncing files + +UPDATE HISTORY: + Written 08/2020 +""" +from __future__ import print_function + +import sys +import os +import re +import time +import getopt +import zipfile +import posixpath +import pyTMD.utilities + +#-- PURPOSE: Download Arctic Ocean Tide Models from the NSF ArcticData archive +def arcticdata_tides(MODEL,DIRECTORY=None,MODE=0o775): + #-- doi for each model + DOI = {} + DOI['AODTM-5'] = '10.18739/A2901ZG3N' + DOI['AOTIM-5'] = '10.18739/A2S17SS80' + DOI['AOTIM-5-2018'] = '10.18739/A21R6N14K' + #-- local subdirectory for each model + LOCAL = {} + LOCAL['AODTM-5'] = 'aodtm5_tmd' + LOCAL['AOTIM-5'] = 'aotim5_tmd' + LOCAL['AOTIM-5-2018'] = 'Arc5km2018' + #-- recursively create directories if non-existent + if not os.access(os.path.join(DIRECTORY,LOCAL[MODEL]), os.F_OK): + os.makedirs(os.path.join(DIRECTORY,LOCAL[MODEL]), MODE) + + #-- build host url for model + resource_map_doi = 'resource_map_doi:{0}'.format(DOI[MODEL]) + HOST = ['https://arcticdata.io','metacat','d1','mn','v2','packages', + pyTMD.utilities.quote_plus(posixpath.join('application','bagit-097')), + pyTMD.utilities.quote_plus(resource_map_doi)] + #-- download zipfile from host + zfile = zipfile.ZipFile(pyTMD.utilities.from_http(HOST)) + print('{0} -->\n'.format(posixpath.join(*HOST))) + #-- find model files within zip file + rx = re.compile('(grid|h[0]?|UV[0]?|Model|xy)_(.*?)',re.VERBOSE) + members = [m for m in zfile.filelist if rx.search(m.filename)] + #-- extract each member + for m in members: + #-- strip directories from member filename + m.filename = posixpath.basename(m.filename) + print('\t{0}\n'.format(os.path.join(DIRECTORY,LOCAL[MODEL],m.filename))) + #-- extract file + zfile.extract(m, path=os.path.join(DIRECTORY,LOCAL[MODEL])) + #-- change permissions mode + os.chmod(os.path.join(DIRECTORY,LOCAL[MODEL],m.filename), MODE) + #-- close the zipfile object + zfile.close() + +#-- PURPOSE: help module to describe the optional input parameters +def usage(): + print('\nHelp: {}'.format(os.path.basename(sys.argv[0]))) + print(' -D X, --directory=X\tWorking data directory') + print(' --tide=X\t\tArctic Ocean tide model to download') + print('\tAODTM-5\n\tAOTIM-5\n\tAOTIM-5-2018') + print(' -M X, --mode=X\t\tPermission mode of files downloaded\n') + +#-- Main program that calls arcticdata_tides() +def main(): + #-- Read the system arguments listed after the program + long_options = ['help','directory=','tide=','mode='] + optlist,arglist = getopt.getopt(sys.argv[1:],'hD:M:',long_options) + + #-- command line parameters + DIRECTORY = os.getcwd() + MODELS = ['AOTIM-5-2018'] + #-- permissions mode of the local directories and files (number in octal) + MODE = 0o775 + for opt, arg in optlist: + if opt in ('-h','--help'): + usage() + sys.exit() + elif opt in ("-D","--directory"): + DIRECTORY = os.path.expanduser(arg) + elif opt in ("--tide",): + MODELS = arg.upper().split(',') + elif opt in ("-M","--mode"): + MODE = int(arg, 8) + + #-- check internet connection before attempting to run program + if pyTMD.utilities.check_connection('https://arcticdata.io'): + for m in MODELS: + arcticdata_tides(m,DIRECTORY=DIRECTORY,MODE=MODE) + +#-- run main program +if __name__ == '__main__': + main() diff --git a/scripts/usap_cats_tides.py b/scripts/usap_cats_tides.py new file mode 100644 index 00000000..b40ee938 --- /dev/null +++ b/scripts/usap_cats_tides.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python +u""" +usap_cats_tides.py +Written by Tyler Sutterley (07/2020) +Download Circum-Antarctic Tidal Simulations from the US Antarctic Program +CATS2008: https://www.usap-dc.org/view/dataset/601235 + +CALLING SEQUENCE: + python usap_cats_tides.py --tide=CATS2008 + +COMMAND LINE OPTIONS: + --help: list the command line options + --directory=X: working data directory + --tide=X: tide model to download + CATS2008 + -M X, --mode=X: Local permissions mode of the files downloaded + +PYTHON DEPENDENCIES: + future: Compatibility layer between Python 2 and Python 3 + https://python-future.org/ + +PROGRAM DEPENDENCIES: + utilities: download and management utilities for syncing files + +UPDATE HISTORY: + Written 08/2020 +""" +from __future__ import print_function + +import sys +import os +import re +import time +import getopt +import zipfile +import posixpath +import pyTMD.utilities + +#-- PURPOSE: Download Circum-Antarctic Tidal Simulations from USAP +def usap_cats_tides(MODEL,DIRECTORY=None,MODE=0o775): + #-- remote subdirectories for each model + REMOTE = {} + REMOTE['CATS2008'] = ['601235','2019-12-19T23:26:43.6Z', + 'CATS2008.zip?dataset_id=601235'] + #-- local subdirectory for each model + LOCAL = {} + LOCAL['CATS2008'] = 'CATS2008' + #-- recursively create directories if non-existent + if not os.access(os.path.join(DIRECTORY,LOCAL[MODEL]), os.F_OK): + os.makedirs(os.path.join(DIRECTORY,LOCAL[MODEL]), MODE) + + #-- download CATS2008 zip file and read as virtual file object + HOST = ['https://www.usap-dc.org','dataset','usap-dc',*REMOTE[MODEL]] + #-- download zipfile from host + zfile = zipfile.ZipFile(pyTMD.utilities.from_http(HOST)) + print('{0} -->\n'.format(posixpath.join(*HOST))) + #-- extract each member + for m in zfile.filelist: + #-- strip directories from member filename + m.filename = posixpath.basename(m.filename) + print('\t{0}\n'.format(os.path.join(DIRECTORY,LOCAL[MODEL],m.filename))) + #-- extract file + zfile.extract(m, path=os.path.join(DIRECTORY,LOCAL[MODEL])) + #-- change permissions mode + os.chmod(os.path.join(DIRECTORY,LOCAL[MODEL],m.filename), MODE) + #-- close the zipfile object + zfile.close() + +#-- PURPOSE: help module to describe the optional input parameters +def usage(): + print('\nHelp: {}'.format(os.path.basename(sys.argv[0]))) + print(' -D X, --directory=X\tWorking data directory') + print(' --tide=X\t\tCircum-Antarctic Tidal Simulation to download') + print('\tCATS2008') + print(' -M X, --mode=X\t\tPermission mode of files downloaded\n') + +#-- Main program that calls usap_cats_tides() +def main(): + #-- Read the system arguments listed after the program + long_options = ['help','directory=','tide=','mode='] + optlist,arglist = getopt.getopt(sys.argv[1:],'hD:M:',long_options) + + #-- command line parameters + DIRECTORY = os.getcwd() + MODELS = ['CATS2008'] + #-- permissions mode of the local directories and files (number in octal) + MODE = 0o775 + for opt, arg in optlist: + if opt in ('-h','--help'): + usage() + sys.exit() + elif opt in ("-D","--directory"): + DIRECTORY = os.path.expanduser(arg) + elif opt in ("--tide",): + MODELS = arg.upper().split(',') + elif opt in ("-M","--mode"): + MODE = int(arg, 8) + + #-- check internet connection before attempting to run program + if pyTMD.utilities.check_connection('https://www.usap-dc.org'): + for m in MODELS: + usap_cats_tides(m,DIRECTORY=DIRECTORY,MODE=MODE) + +#-- run main program +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index e387830d..d8f0e460 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ setup( name='pyTMD', - version='1.0.2.7', + version='1.0.2.8', description='Tide Model Driver to read OTIS, GOT and FES formatted tidal solutions and make tidal predictions', long_description=long_description, long_description_content_type="text/markdown", diff --git a/test/test_download_and_read.py b/test/test_download_and_read.py index 0752c4f9..70e45b14 100644 --- a/test/test_download_and_read.py +++ b/test/test_download_and_read.py @@ -2,6 +2,7 @@ u""" test_download_and_read.py (08/2020) Tests that CATS2008 data can be downloaded from the US Antarctic Program (USAP) +Tests that AOTIM-5-2018 data can be downloaded from the NSF ArcticData server Tests the read program to verify that constituents are being extracted Tests that interpolated results are comparable to Matlab TMD program https://github.com/EarthAndSpaceResearch/TMD_Matlab_Toolbox_v2.5 @@ -17,11 +18,13 @@ UPDATE HISTORY: Updated 08/2020: directly call Matlab program (octave) and compare outputs + compare outputs for both Antarctic (CATS2008) and Arctic (AOTIM-5-2018) will install octave and oct2py in development requirements Updated 08/2020: Download Antarctic tide gauge database and compare with RMS Written 08/2020 """ import os +import re import pytest import inspect import zipfile @@ -52,8 +55,33 @@ def test_download_CATS2008(): assert all([f2 in [f1.filename for f1 in zfile.filelist] for f2 in files]) #-- extract each member for member in zfile.filelist: - local_file = os.path.join(filepath,member.filename) - print('\t{0}\n'.format(local_file)) + #-- strip directories from member filename + member.filename = posixpath.basename(member.filename) + print('\t{0}\n'.format(os.path.join(filepath,member.filename))) + zfile.extract(member, path=filepath) + #-- close the zipfile object + zfile.close() + +#-- PURPOSE: Download AOTIM-5-2018 from NSF ArcticData server +def test_download_AOTIM5_2018(): + #-- build host url for model + resource_map_doi = 'resource_map_doi:{0}'.format('10.18739/A21R6N14K') + HOST = ['https://arcticdata.io','metacat','d1','mn','v2','packages', + pyTMD.utilities.quote_plus(posixpath.join('application','bagit-097')), + pyTMD.utilities.quote_plus(resource_map_doi)] + #-- download zipfile from host + FILE = pyTMD.utilities.from_http(HOST) + zfile = zipfile.ZipFile(FILE) + print('{0} -->\n'.format(posixpath.join(*HOST))) + #-- find model files within zip file + rx = re.compile('(grid|h[0]?|UV[0]?|Model|xy)_(.*?)',re.VERBOSE) + members = [m for m in zfile.filelist if rx.search(m.filename)] + #-- extract each member + for member in members: + #-- strip directories from member filename + member.filename = posixpath.basename(member.filename) + print('\t{0}\n'.format(os.path.join(filepath,member.filename))) + #-- extract file zfile.extract(member, path=filepath) #-- close the zipfile object zfile.close() @@ -66,6 +94,13 @@ def test_download_AntTG(): local = os.path.join(filepath,'AntTG_ocean_height_v1.txt') pyTMD.utilities.from_http(HOST,local=local) +#-- PURPOSE: Download Arctic Tidal Current Atlas list of records +def test_download_Arctic_Tide_Atlas(): + HOST = ['https://arcticdata.io','metacat','d1','mn','v2','object', + 'urn%3Auuid%3Ae3abe2cc-f903-44de-9758-0c6bfc5b66c9'] + local = os.path.join(filepath,'List_of_records.txt') + pyTMD.utilities.from_http(HOST,local=local) + #-- PURPOSE: Test read program that grids and constituents are as expected def test_read_CATS2008(ny=2026,nx=1663): #-- model parameters for CATS2008 @@ -113,7 +148,6 @@ def test_compare_CATS2008(): #-- rewind 1 line count -= 1 #-- iterate over number of stations - AntTG = {} constituents = ['q1','o1','p1','k1','n2','m2','s2','k2'] antarctic_stations = (len(file_contents) - count)//10 stations = [None]*antarctic_stations @@ -213,7 +247,6 @@ def test_verify_CATS2008(): #-- rewind 1 line count -= 1 #-- iterate over number of stations - AntTG = {} antarctic_stations = (len(file_contents) - count)//10 stations = [None]*antarctic_stations shortname = [None]*antarctic_stations @@ -275,9 +308,10 @@ def test_verify_CATS2008(): #-- compute validation data from Matlab TMD program using octave #-- https://github.com/EarthAndSpaceResearch/TMD_Matlab_Toolbox_v2.5 - TMDpath = os.path.join(filepath,'..','TMD_Matlab_Toolbox_v2.5','TMD') + TMDpath = os.path.join(filepath,'..','TMD_Matlab_Toolbox','TMD') octave.addpath(octave.genpath(os.path.normpath(TMDpath))) octave.addpath(filepath) + octave.warning('off', 'all') CFname = os.path.join(filepath,'Model_CATS2008') validation,cons = octave.tmd_tide_pred(CFname,SDtime, station_lat[i],station_lon[i],'z',nout=2) @@ -288,3 +322,87 @@ def test_verify_CATS2008(): difference.mask = (tide.mask | np.isnan(validation)) if not np.all(difference.mask): assert np.all(np.abs(difference) < eps) + +#-- PURPOSE: Tests that interpolated results are comparable to Matlab program +def test_verify_AOTIM5_2018(): + #-- model parameters for AOTIM-5-2018 + grid_file = os.path.join(filepath,'grid_Arc5km2018') + elevation_file = os.path.join(filepath,'h_Arc5km2018') + transport_file = os.path.join(filepath,'UV_Arc5km2018') + GRID = 'OTIS' + EPSG = 'PSNorth' + TYPE = 'z' + + #-- open Arctic Tidal Current Atlas list of records + with open(os.path.join(filepath,'List_of_records.txt'),'r') as f: + file_contents = f.read().splitlines() + #-- skip 2 header rows + count = 2 + #-- iterate over number of stations + arctic_stations = len(file_contents) - count + stations = [None]*arctic_stations + shortname = [None]*arctic_stations + station_lon = np.zeros((arctic_stations)) + station_lat = np.zeros((arctic_stations)) + for s in range(arctic_stations): + line_contents = file_contents[count+s].split() + stations[s] = line_contents[1] + shortname[s] = line_contents[2] + station_lat[s] = np.float(line_contents[10]) + station_lon[s] = np.float(line_contents[11]) + + #-- calculate daily results for a time period + #-- convert time to days since 1992-01-01T00:00:00 + tide_time = np.arange(pyTMD.time.convert_calendar_dates(2000,1,1), + pyTMD.time.convert_calendar_dates(2000,12,31)+1) + #-- serial dates for matlab program (days since 0000-01-01T00:00:00) + SDtime = np.arange(convert_calendar_serial(2000,1,1), + convert_calendar_serial(2000,12,31)+1) + #-- presently not converting times to dynamic times for model comparisons + deltat = np.zeros_like(tide_time) + #-- number of days + ndays = len(tide_time) + + #-- extract amplitude and phase from tide model + amp,ph,D,c = pyTMD.read_tide_model.extract_tidal_constants(station_lon, + station_lat, grid_file, elevation_file, EPSG, TYPE=TYPE, + METHOD='spline', GRID=GRID) + #-- calculate complex phase in radians for Euler's + cph = -1j*ph*np.pi/180.0 + #-- will verify differences between model outputs are within tolerance + eps = np.finfo(np.float16).eps + + #-- compare daily outputs at each station point + invalid_list = ['KS14'] + #-- remove coastal stations from the list + valid_stations=[i for i,s in enumerate(shortname) if s not in invalid_list] + for i in valid_stations: + #-- calculate constituent oscillation for station + hc = amp[i,None,:]*np.exp(cph[i,None,:]) + #-- allocate for out tides at point + tide = np.ma.zeros((ndays)) + tide.mask = np.zeros((ndays),dtype=np.bool) + #-- predict tidal elevations at time and infer minor corrections + tide.mask[:] = np.any(hc.mask) + tide.data[:] = pyTMD.predict_tidal_ts(tide_time, hc, c, + DELTAT=deltat, CORRECTIONS=GRID) + minor = pyTMD.infer_minor_corrections(tide_time, hc, c, + DELTAT=deltat, CORRECTIONS=GRID) + tide.data[:] += minor.data[:] + + #-- compute validation data from Matlab TMD program using octave + #-- https://github.com/EarthAndSpaceResearch/TMD_Matlab_Toolbox_v2.5 + TMDpath = os.path.join(filepath,'..','TMD_Matlab_Toolbox','TMD') + octave.addpath(octave.genpath(os.path.normpath(TMDpath))) + octave.addpath(filepath) + octave.warning('off', 'all') + CFname = os.path.join(filepath,'Model_Arc5km2018') + validation,cons = octave.tmd_tide_pred(CFname,SDtime, + station_lat[i],station_lon[i],'z',nout=2) + + #-- calculate differences between matlab and python version + difference = np.ma.zeros((ndays)) + difference.data[:] = tide.data - validation.T + difference.mask = (tide.mask | np.isnan(validation)) + if not np.all(difference.mask): + assert np.all(np.abs(difference) < eps)