diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..58b1e28 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,34 @@ +name: CI + +on: + push: + branches: + - "master" + pull_request: + branches: + - "*" + schedule: + - cron: "0 13 * * 1" + +jobs: + build: + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.9", "3.10"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: python -m pip install -r requirements.txt + - name: Install Package + run: python -m pip install -e . --no-deps + - name: Test with pytest + run: pytest pangeo_forge_esgf/tests diff --git a/environment.yaml b/environment.yaml index 18e8864..555961d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,6 +1,7 @@ -name: pangeo-forge-esgf +name: pangeo-forge-esgf-dev channels: - conda-forge dependencies: - - python - - aiohttp +- pip +- pip: + - -r requirements.txt diff --git a/pangeo_forge_esgf/tests/test_parsing.py b/pangeo_forge_esgf/tests/test_parsing.py new file mode 100644 index 0000000..8ac22c2 --- /dev/null +++ b/pangeo_forge_esgf/tests/test_parsing.py @@ -0,0 +1,31 @@ +from pangeo_forge_esgf.parsing import parse_instance_ids + + +def test_readme_example(): + # This is possibly flaky (due to the dependence on the ESGF API) + parse_iids = [ + "CMIP6.PMIP.*.*.lgm.*.*.uo.*.*", + "CMIP6.PMIP.*.*.lgm.*.*.vo.*.*", + ] + iids = [] + for piid in parse_iids: + iids.extend(parse_instance_ids(piid)) + iids + + # I expect at least these iids to be parsed + # (there might be new ones that are added at a later point) + expected_iids = [ + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gn.v20191002", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.uo.gn.v20200212", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200212", + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gr1.v20200911", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200909", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.vo.gn.v20200212", + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gn.v20191002", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.vo.gn.v20200212", + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gr1.v20200911", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.vo.gn.v20190710", + ] + + for iid in expected_iids: + assert iid in iids diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5bdc0d9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +aiohttp +requests +pytest diff --git a/setup.cfg b/setup.cfg index 9832028..fe220ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,6 @@ classifiers = Intended Audience :: Science/Research Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Topic :: Scientific/Engineering @@ -23,8 +22,9 @@ keywords = pangeo, data, esgf [options] zip_safe = False -python_requires = >=3.8 +python_requires = >=3.9 packages = find: include_package_data = True install_requires = aiohttp + requests diff --git a/test_script.py b/test_script.py index 3821fa8..6722b61 100644 --- a/test_script.py +++ b/test_script.py @@ -1,3 +1,4 @@ +# %% import asyncio from pangeo_forge_esgf import generate_recipe_inputs_from_iids @@ -7,8 +8,47 @@ "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318", "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120", "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714", + "CMIP6.CMIP.NCC.NorESM2-LM.historical.r1i1p1f1.Omon.vmo.gr.v20190815", + "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318", + "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714", + "CMIP6.CMIP.FIO-QLNM.FIO-ESM-2-0.piControl.r1i1p1f1.Omon.vsf.gn", # this one should not be available. This changes daily. Check the data nodes which are down to find examples. ] recipe_inputs = asyncio.run(generate_recipe_inputs_from_iids(iids)) print("DONE") print(recipe_inputs) + +# %% +# How can I check if a file is available? +# url = 'http://esg1.umr-cnrm.fr/thredds/fileServer/CMIP6_CNRM/PMIP/CNRM-CERFACS/CNRM-CM6-1/lig127k/r1i1p1f2/Amon/tauv/gr/v20200212/tauv_Amon_CNRM-CM6-1_lig127k_r1i1p1f2_gr_185001-209912.nc' +# url = 'http://esgf-nimscmip6.apcc21.org/thredds/fileServer/my_cmip6_dataroot/Historical/R2/aa008p-SImon/CMIP6/CMIP/NIMS-KMA/KACE-1-0-G/historical/r2i1p1f1/SImon/siflsensupbot/gr/v20200130/siflsensupbot_SImon_KACE-1-0-G_historical_r2i1p1f1_gr_185001-201412.nc' +# url = 'http://esgf.ichec.ie' +url = "http://cmip.dess.tsinghua.edu.cn" +# import requests +# from requests import ReadTimeout +# try: +# test = requests.head(url, timeout=5) +# print(test.status_code) +# except ReadTimeout: +# print('Caught timeout. Assuming this file is not available right now') + +# async version +# import asyncio +# import aiohttp +# # async def main(): +# # async with aiohttp.ClientSession() as session: +# # async with session.head(url, timeout=2) as resp: +# # print(resp.status) + + +# async def _check_url(url, session): +# async with session.head(url) as resp: +# return resp.status + +# session = aiohttp.ClientSession() +# status2 = await _check_url(url, session) +# print(status2) + +# 302 is ok? +# %%