From ae4362d82cd1c53b6e69d3d5fc3a16e8c7378625 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 14:57:59 +0200 Subject: [PATCH 1/9] Added import test and fixed dependencies --- .github/workflows/ci.yaml | 36 ++++++++++++++++++ environment.yaml | 7 ++-- pangeo_forge_esgf/tests/test_parsing.py | 31 +++++++++++++++ requirements.txt | 3 ++ test_script.py | 50 ++++++++++++++++++++++--- 5 files changed, 119 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 pangeo_forge_esgf/tests/test_parsing.py create mode 100644 requirements.txt diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a4a4180 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,36 @@ +name: CI + +on: + push: + branches: + - "master" + pull_request: + branches: + - "*" + schedule: + - cron: "0 13 * * 1" + +jobs: + build: + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + python-version: ["3.7", "3.8", "3.9", "3.10"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - run: pip install -r requirements.txt + - id: 'upload-script' + run: python check-open-stores.py + - id: 'save-error-report' + uses: actions/upload-artifact@v2 + with: + name: report + path: report.txt \ No newline at end of file diff --git a/environment.yaml b/environment.yaml index 18e8864..555961d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -1,6 +1,7 @@ -name: pangeo-forge-esgf +name: pangeo-forge-esgf-dev channels: - conda-forge dependencies: - - python - - aiohttp +- pip +- pip: + - -r requirements.txt diff --git a/pangeo_forge_esgf/tests/test_parsing.py b/pangeo_forge_esgf/tests/test_parsing.py new file mode 100644 index 0000000..8e21888 --- /dev/null +++ b/pangeo_forge_esgf/tests/test_parsing.py @@ -0,0 +1,31 @@ +from pangeo_forge_esgf.parsing import parse_instance_ids + +def test_readme_example(): + # This is possibly flaky (due to the dependence on the ESGF API) + from pangeo_forge_esgf.parsing import parse_instance_ids + parse_iids = [ + "CMIP6.PMIP.*.*.lgm.*.*.uo.*.*", + "CMIP6.PMIP.*.*.lgm.*.*.vo.*.*", + ] + iids = [] + for piid in parse_iids: + iids.extend(parse_instance_ids(piid)) + iids + + # I expect at least these iids to be parsed + # (there might be new ones that are added at a later point) + expected_iids = [ + 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gn.v20191002', + 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.uo.gn.v20200212', + 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200212', + 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gr1.v20200911', + 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200909', + 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.vo.gn.v20200212', + 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gn.v20191002', + 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.vo.gn.v20200212', + 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gr1.v20200911', + 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.vo.gn.v20190710' + ] + + for iid in expected_iids: + assert iid in iids \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..94b083a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +aiohttp +requests +pytest \ No newline at end of file diff --git a/test_script.py b/test_script.py index 3821fa8..c14c0e1 100644 --- a/test_script.py +++ b/test_script.py @@ -1,14 +1,54 @@ +# %% import asyncio from pangeo_forge_esgf import generate_recipe_inputs_from_iids iids = [ - # PMIP runs requested by @CommonClimate - "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318", - "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120", - "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714", + #PMIP runs requested by @CommonClimate + 'CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318', + 'CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120', + 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714', + 'CMIP6.CMIP.NCC.NorESM2-LM.historical.r1i1p1f1.Omon.vmo.gr.v20190815', + 'CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318', + 'CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120', + 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714', + 'CMIP6.CMIP.FIO-QLNM.FIO-ESM-2-0.piControl.r1i1p1f1.Omon.vsf.gn',# this one should not be available. This changes daily. Check the data nodes which are down to find examples. ] recipe_inputs = asyncio.run(generate_recipe_inputs_from_iids(iids)) -print("DONE") +print('DONE') print(recipe_inputs) + +# %% +# How can I check if a file is available? +# url = 'http://esg1.umr-cnrm.fr/thredds/fileServer/CMIP6_CNRM/PMIP/CNRM-CERFACS/CNRM-CM6-1/lig127k/r1i1p1f2/Amon/tauv/gr/v20200212/tauv_Amon_CNRM-CM6-1_lig127k_r1i1p1f2_gr_185001-209912.nc' +# url = 'http://esgf-nimscmip6.apcc21.org/thredds/fileServer/my_cmip6_dataroot/Historical/R2/aa008p-SImon/CMIP6/CMIP/NIMS-KMA/KACE-1-0-G/historical/r2i1p1f1/SImon/siflsensupbot/gr/v20200130/siflsensupbot_SImon_KACE-1-0-G_historical_r2i1p1f1_gr_185001-201412.nc' +# url = 'http://esgf.ichec.ie' +url = 'http://cmip.dess.tsinghua.edu.cn' +# import requests +# from requests import ReadTimeout +# try: +# test = requests.head(url, timeout=5) +# print(test.status_code) +# except ReadTimeout: +# print('Caught timeout. Assuming this file is not available right now') + +# async version +# import asyncio +# import aiohttp +# # async def main(): +# # async with aiohttp.ClientSession() as session: +# # async with session.head(url, timeout=2) as resp: +# # print(resp.status) + + +# async def _check_url(url, session): +# async with session.head(url) as resp: +# return resp.status + +# session = aiohttp.ClientSession() +# status2 = await _check_url(url, session) +# print(status2) + +#302 is ok? +# %% \ No newline at end of file From 25b94ef58dbe9c144be00c6cce78190f948c81bb Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 13 Oct 2022 12:59:11 +0000 Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/ci.yaml | 2 +- pangeo_forge_esgf/tests/test_parsing.py | 26 ++++++++++++----------- requirements.txt | 2 +- test_script.py | 28 ++++++++++++------------- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a4a4180..069e840 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,4 +33,4 @@ jobs: uses: actions/upload-artifact@v2 with: name: report - path: report.txt \ No newline at end of file + path: report.txt diff --git a/pangeo_forge_esgf/tests/test_parsing.py b/pangeo_forge_esgf/tests/test_parsing.py index 8e21888..9f6d4da 100644 --- a/pangeo_forge_esgf/tests/test_parsing.py +++ b/pangeo_forge_esgf/tests/test_parsing.py @@ -1,8 +1,10 @@ from pangeo_forge_esgf.parsing import parse_instance_ids + def test_readme_example(): # This is possibly flaky (due to the dependence on the ESGF API) from pangeo_forge_esgf.parsing import parse_instance_ids + parse_iids = [ "CMIP6.PMIP.*.*.lgm.*.*.uo.*.*", "CMIP6.PMIP.*.*.lgm.*.*.vo.*.*", @@ -12,20 +14,20 @@ def test_readme_example(): iids.extend(parse_instance_ids(piid)) iids - # I expect at least these iids to be parsed + # I expect at least these iids to be parsed # (there might be new ones that are added at a later point) expected_iids = [ - 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gn.v20191002', - 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.uo.gn.v20200212', - 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200212', - 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gr1.v20200911', - 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200909', - 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.vo.gn.v20200212', - 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gn.v20191002', - 'CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.vo.gn.v20200212', - 'CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gr1.v20200911', - 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.vo.gn.v20190710' + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gn.v20191002", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.uo.gn.v20200212", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200212", + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gr1.v20200911", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200909", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.vo.gn.v20200212", + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gn.v20191002", + "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.vo.gn.v20200212", + "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gr1.v20200911", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.vo.gn.v20190710", ] for iid in expected_iids: - assert iid in iids \ No newline at end of file + assert iid in iids diff --git a/requirements.txt b/requirements.txt index 94b083a..5bdc0d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ aiohttp requests -pytest \ No newline at end of file +pytest diff --git a/test_script.py b/test_script.py index c14c0e1..6722b61 100644 --- a/test_script.py +++ b/test_script.py @@ -4,27 +4,27 @@ from pangeo_forge_esgf import generate_recipe_inputs_from_iids iids = [ - #PMIP runs requested by @CommonClimate - 'CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318', - 'CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120', - 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714', - 'CMIP6.CMIP.NCC.NorESM2-LM.historical.r1i1p1f1.Omon.vmo.gr.v20190815', - 'CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318', - 'CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120', - 'CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714', - 'CMIP6.CMIP.FIO-QLNM.FIO-ESM-2-0.piControl.r1i1p1f1.Omon.vsf.gn',# this one should not be available. This changes daily. Check the data nodes which are down to find examples. + # PMIP runs requested by @CommonClimate + "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318", + "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714", + "CMIP6.CMIP.NCC.NorESM2-LM.historical.r1i1p1f1.Omon.vmo.gr.v20190815", + "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318", + "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120", + "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714", + "CMIP6.CMIP.FIO-QLNM.FIO-ESM-2-0.piControl.r1i1p1f1.Omon.vsf.gn", # this one should not be available. This changes daily. Check the data nodes which are down to find examples. ] recipe_inputs = asyncio.run(generate_recipe_inputs_from_iids(iids)) -print('DONE') +print("DONE") print(recipe_inputs) # %% # How can I check if a file is available? # url = 'http://esg1.umr-cnrm.fr/thredds/fileServer/CMIP6_CNRM/PMIP/CNRM-CERFACS/CNRM-CM6-1/lig127k/r1i1p1f2/Amon/tauv/gr/v20200212/tauv_Amon_CNRM-CM6-1_lig127k_r1i1p1f2_gr_185001-209912.nc' -# url = 'http://esgf-nimscmip6.apcc21.org/thredds/fileServer/my_cmip6_dataroot/Historical/R2/aa008p-SImon/CMIP6/CMIP/NIMS-KMA/KACE-1-0-G/historical/r2i1p1f1/SImon/siflsensupbot/gr/v20200130/siflsensupbot_SImon_KACE-1-0-G_historical_r2i1p1f1_gr_185001-201412.nc' +# url = 'http://esgf-nimscmip6.apcc21.org/thredds/fileServer/my_cmip6_dataroot/Historical/R2/aa008p-SImon/CMIP6/CMIP/NIMS-KMA/KACE-1-0-G/historical/r2i1p1f1/SImon/siflsensupbot/gr/v20200130/siflsensupbot_SImon_KACE-1-0-G_historical_r2i1p1f1_gr_185001-201412.nc' # url = 'http://esgf.ichec.ie' -url = 'http://cmip.dess.tsinghua.edu.cn' +url = "http://cmip.dess.tsinghua.edu.cn" # import requests # from requests import ReadTimeout # try: @@ -50,5 +50,5 @@ # status2 = await _check_url(url, session) # print(status2) -#302 is ok? -# %% \ No newline at end of file +# 302 is ok? +# %% From 0221630b08e3d33be7486a45b26b64e3da5aba50 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:00:31 +0200 Subject: [PATCH 3/9] fix import typo --- pangeo_forge_esgf/tests/test_parsing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pangeo_forge_esgf/tests/test_parsing.py b/pangeo_forge_esgf/tests/test_parsing.py index 8e21888..58429bc 100644 --- a/pangeo_forge_esgf/tests/test_parsing.py +++ b/pangeo_forge_esgf/tests/test_parsing.py @@ -2,7 +2,6 @@ def test_readme_example(): # This is possibly flaky (due to the dependence on the ESGF API) - from pangeo_forge_esgf.parsing import parse_instance_ids parse_iids = [ "CMIP6.PMIP.*.*.lgm.*.*.uo.*.*", "CMIP6.PMIP.*.*.lgm.*.*.vo.*.*", From f7a5b601e9e2eae5701d12bda7a60e00aa8a07ee Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:06:23 +0200 Subject: [PATCH 4/9] Fix workflow testing --- .github/workflows/ci.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 069e840..8eeb51f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -26,11 +26,7 @@ jobs: - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - run: pip install -r requirements.txt - - id: 'upload-script' - run: python check-open-stores.py - - id: 'save-error-report' - uses: actions/upload-artifact@v2 - with: - name: report - path: report.txt + - name: Install dependencies + run: pip install -r requirements.txt + - name: Test with pytest + run: pytest pangeo_forge_esgf/tests From fb23a351d293dfd1d82d0e7ce0535b23f4f02611 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:08:06 +0200 Subject: [PATCH 5/9] add package install step --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8eeb51f..3156da0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -27,6 +27,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: pip install -r requirements.txt + run: python -m pip install -r requirements.txt + - name: Install Package + run: python -m pip install -e . --no-deps - name: Test with pytest run: pytest pangeo_forge_esgf/tests From a9bbc15751074432a991063022f8847f81658d61 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:10:12 +0200 Subject: [PATCH 6/9] Add py37 --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9832028..e630598 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,7 @@ classifiers = Intended Audience :: Science/Research Programming Language :: Python Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 @@ -23,7 +24,7 @@ keywords = pangeo, data, esgf [options] zip_safe = False -python_requires = >=3.8 +python_requires = >=3.7 packages = find: include_package_data = True install_requires = From 174b645ee33d152e21bb200c0b31bd0d0f1e07d0 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:11:01 +0200 Subject: [PATCH 7/9] only support >=py39 --- setup.cfg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index e630598..71ef01f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,8 +12,6 @@ classifiers = Intended Audience :: Science/Research Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Topic :: Scientific/Engineering @@ -24,7 +22,7 @@ keywords = pangeo, data, esgf [options] zip_safe = False -python_requires = >=3.7 +python_requires = >=3.9 packages = find: include_package_data = True install_requires = From baee2ca9d04f27ac0d7a18df35e9bf4b7c4a4c91 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:11:32 +0200 Subject: [PATCH 8/9] adjust ci matrix to >=py39 --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3156da0..58b1e28 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - python-version: ["3.7", "3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 From 81828ce2ef2aa9f6b450f21d95b0f95ce4cbabc5 Mon Sep 17 00:00:00 2001 From: jbusecke Date: Thu, 13 Oct 2022 15:15:02 +0200 Subject: [PATCH 9/9] forgot install_requires --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 71ef01f..fe220ec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,3 +27,4 @@ packages = find: include_package_data = True install_requires = aiohttp + requests