Merge pull request #8 from jbusecke/dep_bugfix_tests

Added import test and fixed dependencies

.github/workflows/ci.yaml

@@ -0,0 +1,34 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - "master"
+  pull_request:
+    branches:
+      - "*"
+  schedule:
+    - cron: "0 13 * * 1"
+
+jobs:
+  build:
+    defaults:
+      run:
+        shell: bash -l {0}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.9", "3.10"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: python -m pip install -r requirements.txt
+      - name: Install Package
+        run: python -m pip install -e . --no-deps
+      - name: Test with pytest
+        run: pytest pangeo_forge_esgf/tests

environment.yaml

@@ -1,6 +1,7 @@
-name: pangeo-forge-esgf
+name: pangeo-forge-esgf-dev
 channels:
  - conda-forge
 dependencies:
- - python
- - aiohttp
+- pip
+- pip:
+  - -r requirements.txt

pangeo_forge_esgf/tests/test_parsing.py

@@ -0,0 +1,31 @@
+from pangeo_forge_esgf.parsing import parse_instance_ids
+
+
+def test_readme_example():
+    # This is possibly flaky (due to the dependence on the ESGF API)
+    parse_iids = [
+        "CMIP6.PMIP.*.*.lgm.*.*.uo.*.*",
+        "CMIP6.PMIP.*.*.lgm.*.*.vo.*.*",
+    ]
+    iids = []
+    for piid in parse_iids:
+        iids.extend(parse_instance_ids(piid))
+    iids
+
+    # I expect at least these iids to be parsed
+    # (there might be new ones that are added at a later point)
+    expected_iids = [
+        "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gn.v20191002",
+        "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.uo.gn.v20200212",
+        "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200212",
+        "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.uo.gr1.v20200911",
+        "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.uo.gn.v20200909",
+        "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Omon.vo.gn.v20200212",
+        "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gn.v20191002",
+        "CMIP6.PMIP.AWI.AWI-ESM-1-1-LR.lgm.r1i1p1f1.Odec.vo.gn.v20200212",
+        "CMIP6.PMIP.MIROC.MIROC-ES2L.lgm.r1i1p1f2.Omon.vo.gr1.v20200911",
+        "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.lgm.r1i1p1f1.Omon.vo.gn.v20190710",
+    ]
+
+    for iid in expected_iids:
+        assert iid in iids

requirements.txt

@@ -0,0 +1,3 @@
+aiohttp
+requests
+pytest

setup.cfg

@@ -12,7 +12,6 @@ classifiers =
     Intended Audience :: Science/Research
     Programming Language :: Python
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Topic :: Scientific/Engineering
@@ -23,8 +22,9 @@ keywords = pangeo, data, esgf
 
 [options]
 zip_safe = False
-python_requires = >=3.8
+python_requires = >=3.9
 packages = find:
 include_package_data = True
 install_requires =
   aiohttp
+  requests

test_script.py

@@ -1,3 +1,4 @@
+# %%
 import asyncio
 
 from pangeo_forge_esgf import generate_recipe_inputs_from_iids
@@ -7,8 +8,47 @@
     "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318",
     "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120",
     "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714",
+    "CMIP6.CMIP.NCC.NorESM2-LM.historical.r1i1p1f1.Omon.vmo.gr.v20190815",
+    "CMIP6.PMIP.MIROC.MIROC-ES2L.past1000.r1i1p1f2.Amon.tas.gn.v20200318",
+    "CMIP6.PMIP.MRI.MRI-ESM2-0.past1000.r1i1p1f1.Amon.tas.gn.v20200120",
+    "CMIP6.PMIP.MPI-M.MPI-ESM1-2-LR.past2k.r1i1p1f1.Amon.tas.gn.v20210714",
+    "CMIP6.CMIP.FIO-QLNM.FIO-ESM-2-0.piControl.r1i1p1f1.Omon.vsf.gn",  # this one should not be available. This changes daily. Check the data nodes which are down to find examples.
 ]
 
 recipe_inputs = asyncio.run(generate_recipe_inputs_from_iids(iids))
 print("DONE")
 print(recipe_inputs)
+
+# %%
+# How can I check if a file is available?
+# url = 'http://esg1.umr-cnrm.fr/thredds/fileServer/CMIP6_CNRM/PMIP/CNRM-CERFACS/CNRM-CM6-1/lig127k/r1i1p1f2/Amon/tauv/gr/v20200212/tauv_Amon_CNRM-CM6-1_lig127k_r1i1p1f2_gr_185001-209912.nc'
+# url = 'http://esgf-nimscmip6.apcc21.org/thredds/fileServer/my_cmip6_dataroot/Historical/R2/aa008p-SImon/CMIP6/CMIP/NIMS-KMA/KACE-1-0-G/historical/r2i1p1f1/SImon/siflsensupbot/gr/v20200130/siflsensupbot_SImon_KACE-1-0-G_historical_r2i1p1f1_gr_185001-201412.nc'
+# url = 'http://esgf.ichec.ie'
+url = "http://cmip.dess.tsinghua.edu.cn"
+# import requests
+# from requests import ReadTimeout
+# try:
+#     test = requests.head(url, timeout=5)
+#     print(test.status_code)
+# except ReadTimeout:
+#     print('Caught timeout. Assuming this file is not available right now')
+
+# async version
+# import asyncio
+# import aiohttp
+# # async def main():
+# #     async with aiohttp.ClientSession() as session:
+# #         async with session.head(url, timeout=2) as resp:
+# #             print(resp.status)
+
+
+# async def _check_url(url, session):
+#     async with session.head(url) as resp:
+#         return resp.status
+
+# session = aiohttp.ClientSession()
+# status2 = await _check_url(url, session)
+# print(status2)
+
+# 302 is ok?
+# %%