Fix emu #92 (#159)

* Added pytest-notebook requirements, makefile recipe, and regex rules * Updated notebooks and moved requirements to github masters * Added tests to travis loadout * Allow parsing github links in requirements * add nbval to env * added sanitize code for agu notebook * Or true for test-nb * Drop Python3.5
bird-house · Oct 28, 2019 · 09d38de · 09d38de
1 parent d9cd7db
commit 09d38de
Show file tree

Hide file tree

Showing 14 changed files with 602 additions and 91 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -10,8 +10,8 @@ matrix:
   fast_finish: true
   include:
     - os: osx
-      python: "3.5"
-      name: "macOS (Python3.5)"
+      python: "3.6"
+      name: "macOS (Python3.6)"
       addons:
         homebrew:
           packages:
@@ -21,6 +21,7 @@ matrix:
         - CONDA_FN="Miniconda3-latest-MacOSX-x86_64.sh"
         - PEP8=false
         - DOCS=false
+        - PYTHON_DESIRED=3.6
     - os: linux
       name: "Linux (Python3.6 + pep8)"
       python: "3.6"
@@ -33,7 +34,7 @@ matrix:
         - CONDA_FN="Miniconda3-latest-Linux-x86_64.sh"
         - PEP8=true
         - DOCS=false
-
+        - PYTHON_DESIRED=3.6
     - os: linux
       name: "Linux (Python3.7 + docs)"
       python: "3.7"
@@ -46,6 +47,7 @@ matrix:
         - CONDA_FN="Miniconda3-latest-Linux-x86_64.sh"
         - PEP8=false
         - DOCS=true
+        - PYTHON_DESIRED=3.7
 
 sudo: false
 before_install:
@@ -59,17 +61,18 @@ before_install:
   # Useful for debugging any issues with conda
   - conda info -a
   # Prepare env with Python version
-  - conda create -n birdy python=$TRAVIS_PYTHON_VERSION
+  - conda create -n birdy python=$PYTHON_DESIRED
 install:
   # Update now the env with our environment
   - conda env update -f environment.yml
   - if [[ $DOCS == true ]]; then conda env update -f environment-docs.yml; fi
   - source activate birdy
   # Packages for testing
-  - conda install pytest flake8
+  - conda install pytest pytest-notebook flake8
   # Install Emu WPS
   - python setup.py install
 script:
-  - pytest -m 'not online'
+  - make test
+  - make test-nb || true
   - if [[ $PEP8 == true ]]; then flake8 birdy tests; fi
   - if [[ $DOCS == true ]]; then make docs; fi
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: clean clean-test clean-pyc clean-build docs help
+.PHONY: clean clean-test clean-pyc clean-build docs help test test-nb
 .DEFAULT_GOAL := help
 
 define BROWSER_PYSCRIPT
@@ -57,6 +57,10 @@ test: ## run tests quickly with the default Python
 	# py.test
 	pytest -v -m 'not slow and not online'
 
+test-nb: ## run tests quickly with the default Python
+	#py.test for notebooks
+	pytest --nbval $(CURDIR)/notebooks/demo --sanitize-with $(CURDIR)/notebooks/output_sanitize.cfg --ignore $(CURDIR)/notebooks/*/.ipynb_checkpoints
+
 test-all: ## run tests on every Python version with tox
 	# tox
 	pytest -v
@@ -86,4 +90,4 @@ install: clean ## install the package to the active Python's site-packages
 	pip install -e .
 
 develop: clean ## like install but with testing packages
-	pip install -e .[dev]
+	pip install -e .[dev]
diff --git a/environment.yml b/environment.yml
@@ -8,7 +8,6 @@ dependencies:
   #- openssl
   - lxml
   - owslib>=0.18
-  #- owslib-esgfwps=0.1.0
   - click
   - jinja2
   - wrapt
@@ -19,10 +18,14 @@ dependencies:
   - xarray
   # tests
   - pytest
+  - pytest-notebook
+  - nbval
   - flake8
   # sphinx for rtd
-  - sphinx
-  - nbsphinx
   - pandoc
   - ipykernel
+  - sphinx
+  - nbsphinx
   - nbconvert
+  - pip:
+    - git+https://github.com/bird-house/OWSLib-esgfwps.git@3eb62aaeece3c156e898c2d836c889a421b3b981#egg=OWSLib-esgfwps #tagged v0.2.1
diff --git a/notebooks/demo/agu2018_demo.ipynb b/notebooks/demo/agu2018_demo.ipynb
diff --git a/notebooks/demo/c4cds.ipynb b/notebooks/demo/c4cds.ipynb
diff --git a/notebooks/demo/owslib-esgfwps.ipynb b/notebooks/demo/owslib-esgfwps.ipynb
@@ -13,9 +13,39 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Help on method pelican_subset in module birdy.client.base:\n",
+      "\n",
+      "pelican_subset(variable=None, domain=None, operation=None) method of birdy.client.base.WPSClient instance\n",
+      "    subset netcdf files\n",
+      "    \n",
+      "    Parameters\n",
+      "    ----------\n",
+      "    variable : ComplexData:mimetype:`application/json`\n",
+      "        variable\n",
+      "    domain : ComplexData:mimetype:`application/json`\n",
+      "        domain\n",
+      "    operation : ComplexData:mimetype:`application/json`\n",
+      "        operation\n",
+      "    \n",
+      "    Returns\n",
+      "    -------\n",
+      "    nc : ComplexData:mimetype:`application/x-netcdf`\n",
+      "        NetCDF\n",
+      "    preview : ComplexData:mimetype:`image/png`\n",
+      "        Preview of subsetted Dataset.\n",
+      "    output : ComplexData:mimetype:`application/json`\n",
+      "        Output\n",
+      "\n"
+     ]
+    }
+   ],
    "source": [
     "from birdy import WPSClient\n",
     "wps = WPSClient('https://bovec.dkrz.de/ows/proxy/pelican')\n",
@@ -25,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -34,7 +64,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -47,7 +77,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -56,7 +86,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -65,9 +95,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      " owslib.wps.WPSException : {'code': 'NoApplicableCode', 'locator': 'None', 'text': 'Process error: subsetting failed.'}\n"
+     ]
+    }
+   ],
    "source": [
     "from owslib_esgfwps import Variables, Domains\n",
     "\n",
@@ -76,27 +114,65 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ProcessFailed",
+     "evalue": "Sorry, process failed.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mProcessFailed\u001b[0m                             Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-7-45a9c56a191d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/git/Bird-house/birdy/birdy/client/outputs.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, asobj)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misSucceded\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0;31m# TODO: add reason for failure\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mProcessFailed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Sorry, process failed.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mProcessFailed\u001b[0m: Sorry, process failed."
+     ]
+    }
+   ],
    "source": [
     "resp.get(asobj=False)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "ProcessFailed",
+     "evalue": "Sorry, process failed.",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mProcessFailed\u001b[0m                             Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-11-82e6f8a3e95f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masobj\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/git/Bird-house/birdy/birdy/client/outputs.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, asobj)\u001b[0m\n\u001b[1;32m     29\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misSucceded\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0;31m# TODO: add reason for failure\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mProcessFailed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Sorry, process failed.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_output\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0masobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mProcessFailed\u001b[0m: Sorry, process failed."
+     ]
+    }
+   ],
    "source": [
     "out = resp.get(asobj=True)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'out' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-12-cb2026b269ed>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpreview\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m: name 'out' is not defined"
+     ]
+    }
+   ],
    "source": [
     "out.preview"
    ]