Merge pull request #6 from jbusecke/jbusecke_add_specs_tests

Jbusecke add specs tests

.circleci/config.yml

@@ -0,0 +1,47 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: continuumio/miniconda3
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+
+      # - restore_cache:
+      #     keys:
+      #     - v1-dependencies-{{ checksum "environment.yml" }}
+      #     - v1-dependencies-
+
+
+      - run:
+          name: install dependencies
+          command: |
+            # conda env create -q || conda env update -q
+            # source activate adj
+            conda install -qy conda-build anaconda-client pytest pytest-cov
+            conda config --set auto_update_conda no
+            conda info -a
+            conda build conda.recipe --no-test
+            conda install --use-local cmip6_preprocessing
+
+      
+      # - save_cache:
+      #     paths:
+      #       - /opt/conda
+      #     key: v1-dependencies-{{ checksum "environment.yml" }}
+
+
+      - run:
+          name: run tests
+          command: |
+            # source activate adj
+            pytest --color=yes -v --cov=cmip6_preprocessing tests
+            conda install -c conda-forge codecov
+            codecov
+
+      - store_artifacts:
+          path: test-reports
+          destination: test-reports

.codacy.yml

@@ -0,0 +1,8 @@
+---
+exclude_paths:
+  - 'cmip6_preprocessing/_version.py'
+  - 'tests/**/*'
+  - 'tests/*'
+  - 'benchmarks/**/*'
+  - 'setup.py'
+  - 'versioneer.py'

.coveragerc

@@ -0,0 +1,7 @@
+[report]
+omit =
+    setup.py
+    cmip6_preprocessing/__main__.py
+    cmip6_preprocessing/_version.py
+    versioneer.py
+    tests/*

.gitattributes

@@ -0,0 +1,2 @@
+cmip6_preprocessing/_version.py export-subst
+* text=auto eol=lf

.gitignore

@@ -0,0 +1,64 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv python configuration file
+.python-version
+
+*.ipynb_checkpoints

.travis.yml

@@ -0,0 +1,37 @@
+# Config file for automatic testing at travis-ci.org
+
+language: python
+sudo: false # use container based build
+
+matrix:
+  fast_finish: true
+  include:
+  - python: 3.6
+    env: CONDA_ENV=py36
+  - python: 3.7
+    env: CONDA_ENV=py37
+  - python: 3.7
+    env: CONDA_ENV=py37-upstream-master
+
+before_install:
+  - wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh;
+  - bash miniconda.sh -b -p $HOME/miniconda
+  - export PATH="$HOME/miniconda/bin:$PATH"
+  - hash -r
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda info -a 
+  - conda list
+
+install:
+  - conda env create --file ci/environment-$CONDA_ENV.yml
+  - source activate test_env_cmip6_preprocessing
+  - python -c "import cmip6_preprocessing; print(cmip6_preprocessing.__version__)"
+  - pip install -e .
+script:
+  - py.test cmip6_preprocessing -v --cov=cmip6_preprocessing --cov-config .coveragerc --cov-report term-missing
+    #- pytest -v --color=yes --cov=cmip6_preprocessing tests
+
+after_success:
+  - conda install -c conda-forge codecov
+  - codecov

LICENSE

@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2019, Julius Busecke
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

MANIFEST.in

@@ -0,0 +1 @@
+include versioneer.py

README.md

@@ -1,11 +1,22 @@
-# Pangeo cloud demo - Princeton Geoclim Group
+[![Build Status](https://travis-ci.org/jbusecke/cmip6_preprocessing.svg?branch=master)](https://travis-ci.org/jbusecke/cmip6_preprocessing)
+[![codecov](https://codecov.io/gh/jbusecke/cmip6_preprocessing/branch/master/graph/badge.svg)](https://codecov.io/gh/jbusecke/cmip6_preprocessing)
+[![License:MIT](https://img.shields.io/badge/License-MIT-lightgray.svg?style=flt-square)](https://opensource.org/licenses/MIT)
 
-To explore the data available in the cloud check out the full [catalogue](https://pangeo-data.github.io/pangeo-datastore/cmip6_pangeo.html).
+# cmip6_preprocessing
 
-Then head to [pangeo.ocean.io](https://ocean.pangeo.io/) and spawn a jupyterlab notebook
+Frustrated with how 'dirty' CMIP6 data still is? Do you just want to run a simple (or complicated) analysis on various models and end up having to write logic for each seperate case? Then this package is for you.
 
-![login](pics/login.png)
+Developed during the [cmip6-hackathon](https://cmip6hack.github.io/#/) this package provides utility functions that play nicely with [intake-esm](https://github.com/NCAR/intake-esm).
 
-> In order to use [cmip6_preprocessing](https://github.com/jbusecke/cmip6_preprocessing), you need to install it manually whenever you start a new session (it is wiped when you close the session). You can achieve this by either opening a new terminal (+ in the upper left corner) and executing the setup script with `./setup_script`. Or open a notebook and execute `!pip install git+https://github.com/jbusecke/cmip6_preprocessing.git` (You can execute shell commands in notebooks by appending `!`). **In the later case you have to restart the notebook once**.
+We currently support the following functions
 
-Have fun exploring the cloud.
+1. Fix inconsistent naming of dimensions and coordinates
+2. Fix inconsistent values,shape and dataset location of coordinates
+3. Homogenize longitude conventions
+4. Fix inconsistent units
+
+The following issues are under development:
+1. Reconstruct/find grid metrics
+2. Arrange different variables on their respective staggered grid, so they can work seamlessly with [xgcm](https://xgcm.readthedocs.io/en/latest/)
+
+Please check out the [tutorial](notebooks/tutorial.ipynb) for some examples using the [pangeo cloud](ocean.pangeo.io).

catalogs/glade-cmip6.json

@@ -0,0 +1,78 @@
+{
+  "esmcat_version": "0.1.0",
+  "id": "glade-cmip6",
+  "description": "This is an ESM collection for CMIP6 data accessible on the NCAR's GLADE disk storage system in /glade/collections/cmip/CMIP6",
+  "catalog_file": "/glade/collections/cmip/catalog/intake-esm-datastore/catalogs/glade-cmip6.csv.gz",
+  "attributes": [
+    {
+      "column_name": "activity_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_activity_id.json"
+    },
+    {
+      "column_name": "source_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_source_id.json"
+    },
+    {
+      "column_name": "institution_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_institution_id.json"
+    },
+    {
+      "column_name": "experiment_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_experiment_id.json"
+    },
+    { "column_name": "member_id", "vocabulary": "" },
+    {
+      "column_name": "table_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_table_id.json"
+    },
+    { "column_name": "variable_id", "vocabulary": "" },
+    {
+      "column_name": "grid_label",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_grid_label.json"
+    },
+    {
+      "column_name": "version",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "dcpp_start_year",
+      "vocabulary": ""
+    }         
+  ],
+  "assets": {
+    "column_name": "path",
+    "format": "netcdf"
+  },
+  "aggregation_control": {
+    "variable_column_name": "variable_id",
+    "groupby_attrs": [
+      "activity_id",
+      "institution_id",
+      "source_id",
+      "experiment_id",
+      "table_id",
+      "grid_label"
+    ],
+    "aggregations": [
+      {
+        "type": "join_new",
+        "attribute_name": "member_id",
+        "options": { "coords": "minimal", "compat": "override" }
+      },
+    {
+        "type": "join_new",
+        "attribute_name": "dcpp_init_year",
+        "options": { "coords": "minimal", "compat": "override" }
+      },
+      {
+        "type": "join_existing",
+        "attribute_name": "time_range",
+        "options": { "dim": "time" }
+      },
+      {
+        "type": "union",
+        "attribute_name": "variable_id"
+      }
+    ]
+  }
+}

catalogs/pangeo-cmip6.json

@@ -0,0 +1,60 @@
+{
+  "esmcat_version": "0.1.0",
+  "id": "pangeo-cmip6",
+  "description": "This is an ESM collection for CMIP6 Zarr data residing in Pangeo's Google Storage.",
+  "catalog_file": "https://storage.googleapis.com/cmip6/cmip6-zarr-consolidated-stores.csv",
+  "attributes": [
+    {
+      "column_name": "activity_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_activity_id.json"
+    },
+    {
+      "column_name": "source_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_source_id.json"
+    },
+    {
+      "column_name": "institution_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_institution_id.json"
+    },
+    {
+      "column_name": "experiment_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_experiment_id.json"
+    },
+    { "column_name": "member_id", "vocabulary": "" },
+    {
+      "column_name": "table_id",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_table_id.json"
+    },
+    { "column_name": "variable_id", "vocabulary": "" },
+    {
+      "column_name": "grid_label",
+      "vocabulary": "https://raw.githubusercontent.com/WCRP-CMIP/CMIP6_CVs/master/CMIP6_grid_label.json"
+    }
+  ],
+  "assets": {
+    "column_name": "zstore",
+    "format": "zarr"
+  },
+  "aggregation_control": {
+    "variable_column_name": "variable_id",
+    "groupby_attrs": [
+      "activity_id",
+      "institution_id",
+      "source_id",
+      "experiment_id",
+      "table_id",
+      "grid_label"
+    ],
+    "aggregations": [
+      {
+        "type": "join_new",
+        "attribute_name": "member_id",
+        "options": { "coords": "minimal", "compat": "override" }
+      },
+      {
+        "type": "union",
+        "attribute_name": "variable_id"
+      }
+    ]
+  }
+}

ci/environment-py36.yml

@@ -0,0 +1,16 @@
+name: test_env_cmip6_preprocessing
+channels:
+  - conda-forge
+dependencies:
+  - python=3.6
+  - xarray
+  - numpy
+  - pandas
+  - intake-esm
+  - xgcm
+  - pyproj
+  - matplotlib
+  - pip:
+    - codecov
+    - pytest-cov
+    - black