From 39877b0bdfcde97ceaebf229a33a51a058d69556 Mon Sep 17 00:00:00 2001 From: bekozi Date: Fri, 5 May 2017 14:49:29 -0600 Subject: [PATCH] FIX: Fixed some errors related to package upgrades - CRS now needs to check for "towgs84" - Handling netcdf4-python's new datetime objects - Fix quoting for select by SQL --- .travis.yml | 25 ++++-------------- Dockerfile | 10 +++---- environment.yml | 1 + src/ocgis/conv/fiona_.py | 2 ++ src/ocgis/interface/base/crs.py | 17 ++++++++++-- .../interface/base/dimension/temporal.py | 25 +++++++++++------- src/ocgis/interface/base/field.py | 7 ++++- .../test_interface/test_base/test_crs.py | 8 ++++-- .../test_base/test_dimension/test_spatial.py | 18 +++++++------ .../test_base/test_dimension/test_temporal.py | 26 ++++++++++++++++--- .../test_interface/test_base/test_field.py | 5 +--- .../test_ocgis/test_util/test_geom_cabinet.py | 16 ++++++------ src/ocgis/test/test_simple/make_test_data.py | 2 +- src/ocgis/util/environment.py | 1 + src/ocgis/util/pmanager.py | 6 ++--- 15 files changed, 102 insertions(+), 67 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9366e33cf..7a56d54ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,6 @@ language: python sudo: false -env: - global: - - OCGIS_VER="1.3.2" - - ESMPY_VER="7.0.0" - - ICCLIM_VER="4.2.5" - - CONDA_CHANNEL="-c conda-forge -c nesii" - matrix: - - CONDA_PYTHON="python=2.7" - before_install: - wget http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh; - bash miniconda.sh -b -p $HOME/miniconda @@ -21,19 +12,13 @@ before_install: - conda info -a install: - - travis_retry conda create -n test-full $CONDA_CHANNEL $CONDA_PYTHON ocgis=$OCGIS_VER esmpy=$ESMPY_VER nose icclim=$ICCLIM_VER - - source activate test-full + - travis_retry conda env create + - source activate ocgis - conda remove ocgis - python setup.py install - - travis_retry conda create -n test-minimal $CONDA_CHANNEL $CONDA_PYTHON ocgis=$OCGIS_VER nose - - source activate test-minimal - - conda remove rtree cf_units ocgis icclim - - python setup.py install - script: - - source activate test-minimal - - python -c "from ocgis.test import run_simple; run_simple(verbose=False)" - - - source activate test-full + - source activate ocgis - python -c "from ocgis.test import run_all; run_all(verbose=False)" + + - python setup.py test diff --git a/Dockerfile b/Dockerfile index dbd4ce8e1..1321ad766 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,19 +4,19 @@ MAINTAINER ben.koziol@gmail.com RUN apt-get -y update RUN apt-get -y upgrade -RUN apt-get -y install build-essential \ - gfortran +#RUN apt-get -y install build-essential \ +# gfortran RUN apt-get clean RUN conda update -y --all -RUN conda install -y -c nesii ocgis esmpy nose +RUN conda install -y -c conda-forge -c nesii ocgis=1.3.2 esmpy=7.0.0 icclim=4.2.5 nose RUN conda remove -y ocgis -RUN git clone -b next --depth=1 https://github.com/NCPP/ocgis.git +RUN git clone -b v1.3.2 --depth=1 https://github.com/NCPP/ocgis.git RUN cd ocgis && python setup.py install ENV GDAL_DATA /opt/conda/share/gdal -RUN cd && nosetests -a '!slow,!remote,!data' /ocgis/src/ocgis/test +RUN cd && python -c "from ocgis.test import run_simple; run_simple(verbose=False)" RUN rm -r /opt/conda/pkgs/* RUN rm -r /ocgis \ No newline at end of file diff --git a/environment.yml b/environment.yml index 52fb4573c..2248ec534 100644 --- a/environment.yml +++ b/environment.yml @@ -5,6 +5,7 @@ channels: - nesii dependencies: + - python=2.7 - ocgis=1.3.2 - esmpy=7.0.0 - icclim=4.2.5 diff --git a/src/ocgis/conv/fiona_.py b/src/ocgis/conv/fiona_.py index a297747d6..19d7b8838 100644 --- a/src/ocgis/conv/fiona_.py +++ b/src/ocgis/conv/fiona_.py @@ -3,6 +3,7 @@ from types import NoneType import fiona +import netcdftime import numpy as np from ocgis.conv.base import AbstractTabularConverter @@ -23,6 +24,7 @@ class AbstractFionaConverter(AbstractTabularConverter): datetime.date: str} _fiona_type_mapping = {datetime.date: 'str', datetime.datetime: 'str', + netcdftime._netcdftime.datetime: 'str', np.int64: 'int', NoneType: None, np.int32: 'int', diff --git a/src/ocgis/interface/base/crs.py b/src/ocgis/interface/base/crs.py index ebe2bd535..a3135660e 100644 --- a/src/ocgis/interface/base/crs.py +++ b/src/ocgis/interface/base/crs.py @@ -65,7 +65,7 @@ def __init__(self, value=None, proj4=None, epsg=None, name=constants.DEFAULT_COO sr = SpatialReference() sr.ImportFromProj4(to_string(value)) - self.value = from_string(sr.ExportToProj4()) + self.value = from_string(get_proj4_from_spatial_reference(sr)) try: assert self.value != {} @@ -95,7 +95,8 @@ def __str__(self): @property def proj4(self): - return self.sr.ExportToProj4() + ret = get_proj4_from_spatial_reference(self.sr) + return ret @property def sr(self): @@ -781,3 +782,15 @@ def _get_meta_name_(rc_original, key): else: raise return {'meta': meta, 'name': name, 'attrs': attrs} + + +def get_proj4_from_spatial_reference(sr): + """ + :type sr: :class:`osgeo.osr.SpatialReference` + """ + + ret = sr.ExportToProj4() + # Sometimes (for unknown reasons) datum transformations are not returned by PROJ.4. + if ret == '+proj=longlat +datum=WGS84 +no_defs ': + ret = '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ' + return ret diff --git a/src/ocgis/interface/base/dimension/temporal.py b/src/ocgis/interface/base/dimension/temporal.py index 53e5843fd..127bf545d 100644 --- a/src/ocgis/interface/base/dimension/temporal.py +++ b/src/ocgis/interface/base/dimension/temporal.py @@ -8,7 +8,7 @@ import netcdftime import numpy as np -from ocgis import constants, VectorDimension +from ocgis import constants, VectorDimension, env from ocgis.exc import EmptySubsetError, IncompleteSeasonError, CannotFormatTimeError, ResolutionError from ocgis.util.helpers import get_is_date_between, iter_array, get_none_or_slice @@ -132,7 +132,7 @@ def get_datetime(self, arr): # If there are month units, call the special procedure to convert those to datetime objects. if not self._has_months_units: arr = np.atleast_1d(nc.num2date(arr, str(self.units), calendar=self.calendar)) - dt = datetime.datetime + dt = get_datetime_or_netcdftime for idx, t in iter_array(arr, return_value=True): # Attempt to convert times to datetime objects. try: @@ -499,6 +499,7 @@ def _get_attrs_(dt): except TypeError: new_value[idx]['months'] = grouping[idx][0] sel = value[dgrp][:, (0, 2)] + new_bounds[idx, :] = [sel.min(), sel.max()] new_bounds = np.atleast_2d(new_bounds).reshape(-1, 2) @@ -683,7 +684,7 @@ def get_datetime_conversion_state(archetype): :rtype: bool """ - if isinstance(archetype, (datetime.datetime, netcdftime.datetime)): + if isinstance(archetype, (datetime.datetime, netcdftime.datetime, netcdftime._netcdftime.datetime)): ret = False else: ret = True @@ -750,12 +751,18 @@ def get_datetime_from_template_time_units(vec): return fill -def get_datetime_or_netcdftime(year, month, day, **kwargs): - try: - ret = datetime.datetime(year, month, day, **kwargs) - except ValueError: - # Assume the datetime object is not compatible with the arguments. Return a netcdftime object. - ret = netcdftime.datetime(year, month, day, **kwargs) +def get_datetime_or_netcdftime(*args, **kwargs): + if env.PREFER_NETCDFTIME: + try: + ret = netcdftime.datetime(*args, **kwargs) + except ValueError: + # Assume the datetime object is not compatible with the arguments. Return a netcdftime object. + ret = datetime.datetime(*args, **kwargs) + else: + try: + ret = datetime.datetime(*args, **kwargs) + except ValueError: + ret = netcdftime.datetime(*args, **kwargs) return ret diff --git a/src/ocgis/interface/base/field.py b/src/ocgis/interface/base/field.py index d5533678b..792eaca8b 100644 --- a/src/ocgis/interface/base/field.py +++ b/src/ocgis/interface/base/field.py @@ -4,6 +4,7 @@ from copy import copy, deepcopy import fiona +import netcdftime import numpy as np from shapely.geometry import mapping from shapely.geometry.multipoint import MultiPoint @@ -206,7 +207,11 @@ def get_fiona_dict(field, arch): fproperties = OrderedDict() fconvert = {} for k, v in arch.iteritems(): - ftype = AbstractFionaConverter.get_field_type(type(v)) + if isinstance(v, netcdftime._netcdftime.datetime): + type_v = netcdftime._netcdftime.datetime + else: + type_v = type(v) + ftype = AbstractFionaConverter.get_field_type(type_v) fproperties[k] = 'int' if ftype is None else ftype if ftype == 'str': fconvert[k] = str diff --git a/src/ocgis/test/test_ocgis/test_interface/test_base/test_crs.py b/src/ocgis/test/test_ocgis/test_interface/test_base/test_crs.py index 7aefbf3c9..2d32a896a 100644 --- a/src/ocgis/test/test_ocgis/test_interface/test_base/test_crs.py +++ b/src/ocgis/test/test_ocgis/test_interface/test_base/test_crs.py @@ -39,8 +39,12 @@ def test_init(self): else: raise self.assertEqual(crs.proj4, '+proj=longlat +ellps=WGS84 +towgs84=0,0,0,0,0,0,0 +no_defs ') - self.assertDictEqual(crs.value, - {'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}) + try: + self.assertDictEqual(crs.value, + {'no_defs': True, 'ellps': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}) + except AssertionError: + self.assertDictEqual(crs.value, + {'no_defs': True, 'datum': 'WGS84', 'proj': 'longlat', 'towgs84': '0,0,0,0,0,0,0'}) if prev_crs is not None: self.assertEqual(crs, prev_crs) prev_crs = deepcopy(crs) diff --git a/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_spatial.py b/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_spatial.py index 57c75baa1..5afb34839 100644 --- a/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_spatial.py +++ b/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_spatial.py @@ -806,11 +806,13 @@ def test_update_crs_geom_combinations(self): sdim.update_crs(to_crs) if k.with_polygon: - self.assertEqual(sdim.geom.polygon.value[2, 2].bounds, - (130734.585229303, -832179.0855220362, 220974.77455120225, -719113.1357226598)) + actual = sdim.geom.polygon.value[2, 2].bounds + desired = (130734.585229303, -832179.0855220362, 220974.77455120225, -719113.1357226598) + if k.with_point: - self.assertEqual(sdim.geom.point.value[2, 2].bounds, - (175552.29305101855, -775779.6191590576, 175552.29305101855, -775779.6191590576)) + actual = sdim.geom.point.value[2, 2].bounds + desired = (175552.29305101855, -775779.6191590576, 175552.29305101855, -775779.6191590576) + self.assertNumpyAllClose(np.array(actual), np.array(desired)) def test_update_crs_grid_combinations(self): """Test CRS is updated as expected with different types of grids.""" @@ -859,12 +861,12 @@ def test_update_crs_grid_combinations(self): self.assertIsNone(sdim.grid.col) try: - self.assertEqual(sdim.geom.polygon.value[2, 2].bounds, - (130734.585229303, -832179.0855220362, 220974.77455120225, -719113.1357226598)) + desired = np.array([130734.585229303, -832179.0855220362, 220974.77455120225, -719113.1357226598]) + self.assertNumpyAllClose(np.array(sdim.geom.polygon.value[2, 2].bounds), desired) except AttributeError: self.assertFalse(k.with_corners) - self.assertEqual(sdim.geom.point.value[2, 2].bounds, - (175552.29305101855, -775779.6191590576, 175552.29305101855, -775779.6191590576)) + desired = np.array([175552.29305101855, -775779.6191590576, 175552.29305101855, -775779.6191590576]) + self.assertNumpyAllClose(np.array(sdim.geom.point.value[2, 2].bounds), desired) sdim.update_crs(from_crs) diff --git a/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_temporal.py b/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_temporal.py index 08f253969..61ad89e3d 100644 --- a/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_temporal.py +++ b/src/ocgis/test/test_ocgis/test_interface/test_base/test_dimension/test_temporal.py @@ -16,7 +16,8 @@ from ocgis.interface.base.dimension.temporal import TemporalDimension, get_is_interannual, get_sorted_seasons, \ get_time_regions, iter_boolean_groups_from_time_regions, get_datetime_conversion_state, \ get_datetime_from_months_time_units, get_difference_in_months, get_num_from_months_time_units, \ - get_origin_datetime_from_months_units, get_datetime_from_template_time_units, TemporalGroupDimension + get_origin_datetime_from_months_units, get_datetime_from_template_time_units, TemporalGroupDimension, \ + get_datetime_or_netcdftime from ocgis.test.base import TestBase, nc_scope, attr from ocgis.util.helpers import get_date_list from ocgis.util.itester import itr_products_keywords @@ -147,6 +148,14 @@ def get_temporal_dimension(self, add_bounds=True, start=None, stop=None, days=1, bounds[:, 1] = upper else: bounds = None + + dates = [get_datetime_or_netcdftime(d.year, d.month, d.day, hour=d.hour) for d in dates] + if add_bounds: + bounds_shape = bounds.shape + bounds = bounds.flatten() + bounds = np.array([get_datetime_or_netcdftime(d.year, d.month, d.day, hour=d.hour) for d in bounds]) + bounds = bounds.reshape(*bounds_shape) + td = TemporalDimension(value=dates, bounds=bounds, name=name, format_time=format_time) return td @@ -357,7 +366,10 @@ def test_get_datetime(self): narr = date2num(ndts, units, calendar=calendar) td = TemporalDimension(value=narr, units=units, calendar=calendar) res = td.get_datetime(td.value) - self.assertTrue(all([isinstance(element, ndt) for element in res.flat])) + try: + self.assertTrue(all([isinstance(element, ndt) for element in res.flat])) + except AssertionError: + self.assertTrue(all([isinstance(element, netcdftime._netcdftime.datetime) for element in res.flat])) # test with template units td = self.get_template_units() @@ -373,7 +385,10 @@ def test_getiter(self): to_test = (values['day'], values['month'], values['year']) try: self.assertTrue(all([element is not None for element in to_test])) - self.assertIsInstance(values['time'], dt) + try: + self.assertIsInstance(values['time'], dt) + except: + self.assertIsInstance(values['time'], netcdftime._netcdftime.datetime) except AssertionError: self.assertTrue(all([element is None for element in to_test])) self.assertIsInstance(values['time'], float) @@ -388,7 +403,8 @@ def test_get_numtime(self): def test_get_grouping(self): td = self.get_temporal_dimension() - td = td.get_between(datetime.datetime(1900, 1, 1), datetime.datetime(1900, 12, 31, 23, 59)) + lower_upper = [datetime.datetime(1900, 1, 1), datetime.datetime(1900, 12, 31, 23, 59)] + td = td.get_between(*lower_upper) tgd = td.get_grouping(['year']) self.assertEqual(tgd.value, np.array([datetime.datetime(1900, 7, 1)])) @@ -397,6 +413,8 @@ def test_get_grouping(self): stop = 719.9375 step = 0.125 values = np.arange(start, stop + step, step) + values = num2date(values, 'days since 1960-01-01', calendar='360_day') + td = TemporalDimension(value=values, calendar='360_day', units='days since 1960-01-01') for g in CalcGrouping.iter_possible(): tgd = td.get_grouping(g) diff --git a/src/ocgis/test/test_ocgis/test_interface/test_base/test_field.py b/src/ocgis/test/test_ocgis/test_interface/test_base/test_field.py index debd777e6..3e8413bf2 100644 --- a/src/ocgis/test/test_ocgis/test_interface/test_base/test_field.py +++ b/src/ocgis/test/test_ocgis/test_interface/test_base/test_field.py @@ -895,7 +895,4 @@ def test_init(self): df = DerivedField(variables=mu, temporal=tgd, spatial=field.spatial, level=field.level, realization=field.realization) self.assertIsInstance(df, Field) - self.assertIsInstance(df.temporal.value[0], datetime.datetime) - self.assertEqual(df.temporal.value.tolist(), - [datetime.datetime(2000, 1, 16, 0, 0), datetime.datetime(2000, 2, 16, 0, 0)]) - self.assertEqual(df.temporal.bounds[1, 1], datetime.datetime(2000, 3, 1, 0, 0)) + self.assertEqual(df.temporal.value_numtime.tolist(), [730136.0, 730167.0]) diff --git a/src/ocgis/test/test_ocgis/test_util/test_geom_cabinet.py b/src/ocgis/test/test_ocgis/test_util/test_geom_cabinet.py index b80c1dda1..1f3076923 100644 --- a/src/ocgis/test/test_ocgis/test_util/test_geom_cabinet.py +++ b/src/ocgis/test/test_ocgis/test_util/test_geom_cabinet.py @@ -87,7 +87,7 @@ def test_as_spatial_dimension_points(self): def test_iter(self): # test with a select statement - sci = GeomCabinetIterator(key='state_boundaries', select_sql_where='STATE_NAME in ("Wisconsin", "Vermont")') + sci = GeomCabinetIterator(key='state_boundaries', select_sql_where="STATE_NAME in ('Wisconsin', 'Vermont')") for row in sci: self.assertIn(row['properties']['STATE_NAME'], ("Wisconsin", "Vermont")) @@ -116,7 +116,7 @@ def test_len(self): sci = GeomCabinetIterator(path=path, select_uid=[16, 19]) self.assertEqual(len(sci), 2) - sci = GeomCabinetIterator(key='state_boundaries', select_sql_where='STATE_NAME = "Vermont"') + sci = GeomCabinetIterator(key='state_boundaries', select_sql_where="STATE_NAME = 'Vermont'") self.assertEqual(len(sci), 1) def test_iteration_by_path(self): @@ -165,7 +165,7 @@ def test_get_features_object(self): path = self.get_shapefile_path_with_no_ugid() keywords = dict(uid=[None, 'ID'], select_uid=[None, [8, 11, 13]], - select_sql_where=[None, 'STATE_NAME = "Wisconsin"']) + select_sql_where=[None, "STATE_NAME = 'Wisconsin'"]) for k in self.iter_product_keywords(keywords): ds = ogr.Open(path) @@ -209,7 +209,7 @@ def _run_(s, func): finally: ds.Destroy() - s = 'STATE_NAME in ("Wisconsin", "Vermont")' + s = "STATE_NAME in ('Wisconsin', 'Vermont')" def f(obj): self.assertEqual(len(obj), 2) @@ -217,14 +217,14 @@ def f(obj): _run_(s, f) - s = 'STATE_NAME in ("Wisconsin", "Vermont") and STATE_ABBR in ("NV", "OH")' + s = "STATE_NAME in ('Wisconsin', 'Vermont') and STATE_ABBR in ('NV', 'OH')" def f(obj): self.assertEqual(len(obj), 0) _run_(s, f) - s = 'STATE_NAME in ("Wisconsin", "Vermont") or STATE_ABBR in ("NV", "OH")' + s = "STATE_NAME in ('Wisconsin', 'Vermont') or STATE_ABBR in ('NV', 'OH')" def f(obj): self.assertEqual(len(obj), 4) @@ -311,7 +311,7 @@ def test_iter_geoms(self): def test_iter_geoms_select_sql_where(self): sc = GeomCabinet() - sql = 'STATE_NAME = "New Hampshire"' + sql = "STATE_NAME = 'New Hampshire'" self.assertEqual(len(list(sc.iter_geoms('state_boundaries', select_sql_where=sql))), 1) def test_iter_geoms_select_ugid(self): @@ -325,7 +325,7 @@ def test_sql_subset(self): sc = GeomCabinet() path = sc.get_shp_path('state_boundaries') ds = ogr.Open(path) - ret = ds.ExecuteSQL('select * from state_boundaries where state_name = "New Jersey"') + ret = ds.ExecuteSQL("select * from state_boundaries where state_name = 'New Jersey'") ret.ResetReading() self.assertEqual(len(ret), 1) diff --git a/src/ocgis/test/test_simple/make_test_data.py b/src/ocgis/test/test_simple/make_test_data.py index b6769d6b9..d0a144af2 100644 --- a/src/ocgis/test/test_simple/make_test_data.py +++ b/src/ocgis/test/test_simple/make_test_data.py @@ -84,7 +84,7 @@ def write(self): timevec_bnds[idx, 0] = tv - delta timevec_bnds[idx, 1] = tv + delta - # ## make the level vector + # ## make the level vector # levelvec = np.array([50,150]) # levelvec_bounds = np.array([[0,100],[100,200]]) diff --git a/src/ocgis/util/environment.py b/src/ocgis/util/environment.py index 681d48e6f..802f703aa 100644 --- a/src/ocgis/util/environment.py +++ b/src/ocgis/util/environment.py @@ -72,6 +72,7 @@ def __init__(self): self.NETCDF_FILE_FORMAT = EnvParm('NETCDF_FILE_FORMAT', constants.NETCDF_DEFAULT_DATA_MODEL, formatter=str) self.NP_INT = EnvParm('NP_INT', constants.DEFAULT_NP_INT) self.NP_FLOAT = EnvParm('NP_FLOAT', constants.DEFAULT_NP_FLOAT) + self.PREFER_NETCDFTIME = EnvParm('PREFER_NETCDFTIME', True, formatter=self._format_bool_) from ocgis.interface.base.crs import CFWGS84 diff --git a/src/ocgis/util/pmanager.py b/src/ocgis/util/pmanager.py index 7feaf95ec..50baa4050 100644 --- a/src/ocgis/util/pmanager.py +++ b/src/ocgis/util/pmanager.py @@ -42,9 +42,9 @@ def run(self): except IndexError: self.join() break - # codes = [bool(p.exitcode) for p in self.procs] - # if any(codes): - # raise(RuntimeError('{0} processes had a non-zero exit status.'.format(sum(codes)))) + # codes = [bool(p.exitcode) for p in self.procs] + # if any(codes): + # raise(RuntimeError('{0} processes had a non-zero exit status.'.format(sum(codes)))) def alive(self): count = sum([p.is_alive() for p in self.procs])