From b9f69d1453b567fa169b560989728c044cbb5276 Mon Sep 17 00:00:00 2001 From: thequackdaddy Date: Sat, 8 Apr 2017 07:07:52 +0000 Subject: [PATCH] Getting closer on pickling * Now run subset and clean the eval_env after removing bare funcalls * Had some straggling tuples * Altered test so that stateful_transforms won't be saved in eval_env * Removed the clean method on EvalEnvironment --- patsy/contrasts.py | 17 ++--- patsy/design_info.py | 6 -- patsy/eval.py | 64 ++++++------------ patsy/test_pickling.py | 13 ---- patsy/util.py | 2 +- pickle_testcases/0.5/contrast_matrix.pickle | Bin 224 -> 279 bytes pickle_testcases/0.5/evalenv_pickle.pickle | Bin 128 -> 0 bytes pickle_testcases/0.5/evalenv_simple.pickle | Bin 67 -> 0 bytes .../0.5/evalenv_transform_bs.pickle | Bin 60 -> 0 bytes .../0.5/evalenv_transform_categorical.pickle | Bin 89 -> 0 bytes .../0.5/evalenv_transform_cc.pickle | Bin 60 -> 0 bytes .../0.5/evalenv_transform_center.pickle | Bin 60 -> 0 bytes .../0.5/evalenv_transform_cr.pickle | Bin 60 -> 0 bytes .../0.5/evalenv_transform_scale.pickle | Bin 60 -> 0 bytes .../0.5/evalenv_transform_standardize.pickle | Bin 60 -> 0 bytes .../0.5/evalenv_transform_te.pickle | Bin 60 -> 0 bytes pickle_testcases/0.5/subterm_info.pickle | Bin 361 -> 392 bytes pickle_testcases/0.5/transform_center.pickle | Bin 218 -> 227 bytes .../transform_standardize_norescale.pickle | Bin 287 -> 304 bytes .../0.5/transform_standardize_rescale.pickle | Bin 287 -> 304 bytes .../0.5/varlookupdict_simple.pickle | Bin 80 -> 0 bytes 21 files changed, 26 insertions(+), 76 deletions(-) delete mode 100644 pickle_testcases/0.5/evalenv_pickle.pickle delete mode 100644 pickle_testcases/0.5/evalenv_simple.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_bs.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_categorical.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_cc.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_center.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_cr.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_scale.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_standardize.pickle delete mode 100644 pickle_testcases/0.5/evalenv_transform_te.pickle delete mode 100644 pickle_testcases/0.5/varlookupdict_simple.pickle diff --git a/patsy/contrasts.py b/patsy/contrasts.py index 4173f2e..4d13d20 100644 --- a/patsy/contrasts.py +++ b/patsy/contrasts.py @@ -49,20 +49,13 @@ def _repr_pretty_(self, p, cycle): def __getstate__(self): - return (0, self.matrix, self.column_suffixes) + return {'version': 0, 'matrix': self.matrix, + 'column_suffixes': self.column_suffixes} def __setstate__(self, pickle): - version, matrix, column_suffixes = pickle - check_pickle_version(version, 0, name=self.__class__.__name__) - self.matrix = matrix - self.column_suffixes = column_suffixes - - def __eq__(self, other): - if self.column_suffixes != other.column_suffixes: - return False - if not np.array_equal(self.matrix, other.matrix): - return False - return True + check_pickle_version(pickle['version'], 0, name=self.__class__.__name__) + self.matrix = pickle['matrix'] + self.column_suffixes = pickle['column_suffixes'] def test_ContrastMatrix(): diff --git a/patsy/design_info.py b/patsy/design_info.py index 885503f..b6e2fa9 100644 --- a/patsy/design_info.py +++ b/patsy/design_info.py @@ -737,12 +737,6 @@ def __init__(self, name): def name(self): return self._name - def __eq__(self, other): - return self.__dict__ == other.__dict__ - - def __hash__(self): - return hash((_MockFactor, str(self._name))) - def test_DesignInfo(): from nose.tools import assert_raises diff --git a/patsy/eval.py b/patsy/eval.py index c853988..cab3dc6 100644 --- a/patsy/eval.py +++ b/patsy/eval.py @@ -25,6 +25,7 @@ from patsy.tokens import (pretty_untokenize, normalize_token_spacing, python_tokenize) from patsy.compat import call_and_wrap_exc +from nose.tools import assert_raises def _all_future_flags(): flags = 0 @@ -71,13 +72,8 @@ def get(self, key, default=None): def __repr__(self): return "%s(%r)" % (self.__class__.__name__, self._dicts) - def __getstate__(self): - return (0, self._dicts) + __getstate__ = no_pickling - def __setstate__(self, pickle): - version, dicts = pickle - check_pickle_version(version, 0, name=self.__class__.__name__) - self._dicts = dicts def test_VarLookupDict(): d1 = {"a": 1} @@ -262,27 +258,15 @@ def __hash__(self): tuple(self._namespace_ids()))) def __getstate__(self): - self.clean() + # self.clean() namespaces = self._namespaces namespaces = _replace_un_pickleable(namespaces) - return (0, namespaces, self.flags) + return {'version': 0, 'namespaces': namespaces, 'flags': self.flags} def __setstate__(self, pickle): - version, namespaces, flags = pickle - check_pickle_version(version, 0, self.__class__.__name__) - self.flags = flags - self._namespaces = _return_un_pickleable(namespaces) - - def clean(self): - """The EvalEnvironment doesn't need the stateful transformation - functions once the design matrix has been built. This will delete - it. Called by __getstate__ to prepare for pickling.""" - namespaces = [] - for namespace in self._namespaces: - ns = {key: namespace[key] for key in six.iterkeys(namespace) if not - hasattr(namespace[key], '__patsy_stateful_transform__')} - namespaces.append(ns) - self._namespaces = namespaces + check_pickle_version(pickle['version'], 0, self.__class__.__name__) + self.flags = pickle['flags'] + self._namespaces = _return_un_pickleable(pickle['namespaces']) class ObjectHolder(object): @@ -504,21 +488,6 @@ def test_EvalEnvironment_eq(): assert env3 != env4 -def test_EvalEnvironment_clean(): - from patsy.state import center, standardize - from patsy.splines import bs - - env1 = EvalEnvironment([{'center': center}]) - env2 = EvalEnvironment([{'standardize': standardize}]) - env3 = EvalEnvironment([{'bs': bs}]) - env1.clean() - env2.clean() - env3.clean() - - env1._namespaces == [{}] - env2._namespaces == [{}] - env3._namespaces == [{}] - _builtins_dict = {} six.exec_("from patsy.builtins import *", {}, _builtins_dict) # This is purely to make the existence of patsy.builtins visible to systems @@ -576,10 +545,6 @@ def memorize_passes_needed(self, state, eval_env): eval_env = eval_env.with_outer_namespace(_builtins_dict) env_namespace = eval_env.namespace - subset_names = [name for name in ast_names(self.code) - if name in env_namespace] - eval_env = eval_env.subset(subset_names) - state["eval_env"] = eval_env # example code: == "2 * center(x)" i = [0] @@ -596,6 +561,12 @@ def new_name_maker(token): # example eval_code: == "2 * _patsy_stobj0__center__.transform(x)" eval_code = replace_bare_funcalls(self.code, new_name_maker) state["eval_code"] = eval_code + + subset_names = [name for name in ast_names(eval_code) + if name in env_namespace] + eval_env = eval_env.subset(subset_names) + state["eval_env"] = eval_env + # paranoia: verify that none of our new names appeared anywhere in the # original code if has_bare_variable_reference(state["transforms"], self.code): @@ -716,7 +687,10 @@ def test_EvalFactor_memorize_passes_needed(): print(state) assert passes == 2 for name in ["foo", "bar", "quux"]: - assert state["eval_env"].namespace[name] is locals()[name] + # name should be locally defined, but since its a stateful_transform, + # its unnecessary to keep it in eval_env + assert name in locals() + assert_raises(KeyError, state["eval_env"].namespace.__getitem__, name) for name in ["w", "x", "y", "z", "e", "state"]: assert name not in state["eval_env"].namespace assert state["transforms"] == {"_patsy_stobj0__foo__": "FOO-OBJ", @@ -772,7 +746,9 @@ def test_EvalFactor_end_to_end(): print(passes) print(state) assert passes == 2 - assert state["eval_env"].namespace["foo"] is foo + # We don't want to save the stateful transforms in the eval_env, actually. + # Just + assert_raises(KeyError, state["eval_env"].namespace.__getitem__, 'foo') for name in ["x", "y", "e", "state"]: assert name not in state["eval_env"].namespace import numpy as np diff --git a/patsy/test_pickling.py b/patsy/test_pickling.py index fb34667..b666d6d 100644 --- a/patsy/test_pickling.py +++ b/patsy/test_pickling.py @@ -42,19 +42,6 @@ def _unwrap_stateful_function(function, *args, **kwargs): pickling_testcases = { "evalfactor_simple": EvalFactor("a+b"), - "varlookupdict_simple": VarLookupDict([{"a": 1}, {"a": 2, "b": 3}]), - "evalenv_simple": EvalEnvironment([{"a": 1}]), - "evalenv_transform_center": EvalEnvironment([{'center': center}]), - "evalenv_transform_scale": EvalEnvironment([{'scale': scale}]), - "evalenv_transform_standardize": EvalEnvironment([{ - 'standardize': standardize - }]), - "evalenv_transform_categorical": EvalEnvironment([{'C': C}]), - "evalenv_transform_bs": EvalEnvironment([{'cs': bs}]), - "evalenv_transform_te": EvalEnvironment([{'te': te}]), - "evalenv_transform_cr": EvalEnvironment([{'cs': cr}]), - "evalenv_transform_cc": EvalEnvironment([{'cc': cc}]), - "evalenv_pickle": EvalEnvironment([{'np': np}]), "term": Term([1, 2, 1]), "contrast_matrix": ContrastMatrix([[1, 0], [0, 1]], ["a", "b"]), "subterm_info": si, diff --git a/patsy/util.py b/patsy/util.py index 70909c8..afcbc0c 100644 --- a/patsy/util.py +++ b/patsy/util.py @@ -29,7 +29,7 @@ import numpy as np import six from six.moves import cStringIO as StringIO -from patsy.compat import optional_dep_ok +from .compat import optional_dep_ok try: import pandas diff --git a/pickle_testcases/0.5/contrast_matrix.pickle b/pickle_testcases/0.5/contrast_matrix.pickle index 0d63acd1d751148ab69e119a9c00cdade3449416..44ac6f109fa5789bbd4e5cffafcf05def4d9ef4d 100644 GIT binary patch delta 131 zcmaFBIGu^Lfn_QW<3v_HiP|X|J?v$vMa7xQeqL^BUdfcnQ#2c=cr(OKsh!fp zlvgk%gQ17VKPf9Uxx^zsCndEAqM&w)Mh{DNW?ssa9=6>4l+v8kDH%LHta+um1(j2J VSn?8cfdXu$NyUlY4DC~r^Z<%&FTnr+ diff --git a/pickle_testcases/0.5/evalenv_simple.pickle b/pickle_testcases/0.5/evalenv_simple.pickle deleted file mode 100644 index 7b8258a0eb3d9eb7b09ce279adf20ab2bc0d595e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 67 zcmZo*nQFlR0XQeqL^BUdfcnQ#2c=cr(OKsh!fp Rm^j6ou{hD2p?yk{9stk*7a0Hm diff --git a/pickle_testcases/0.5/evalenv_transform_bs.pickle b/pickle_testcases/0.5/evalenv_transform_bs.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/evalenv_transform_categorical.pickle b/pickle_testcases/0.5/evalenv_transform_categorical.pickle deleted file mode 100644 index 33d403c63b468c16bd43a4febbf3b4d614236184..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 89 zcmZo*nd-*?0XQeqL^BUdfcnQ#2c=cr(OKsh!fp i=scxI5TZIcu_QG;zbG>qs40VO@|5C4Z-(|MNqPWF3m|9! diff --git a/pickle_testcases/0.5/evalenv_transform_cc.pickle b/pickle_testcases/0.5/evalenv_transform_cc.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/evalenv_transform_center.pickle b/pickle_testcases/0.5/evalenv_transform_center.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/evalenv_transform_cr.pickle b/pickle_testcases/0.5/evalenv_transform_cr.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/evalenv_transform_scale.pickle b/pickle_testcases/0.5/evalenv_transform_scale.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/evalenv_transform_standardize.pickle b/pickle_testcases/0.5/evalenv_transform_standardize.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/evalenv_transform_te.pickle b/pickle_testcases/0.5/evalenv_transform_te.pickle deleted file mode 100644 index 1d48cbd684bd13abf9c9d4aa0e85da001e5316b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 60 zcmZo*nQF)Y0XQeqL^BUdfcnQ#2c=cr(OKshyJO K&Cot2Ne=)szZB5` diff --git a/pickle_testcases/0.5/subterm_info.pickle b/pickle_testcases/0.5/subterm_info.pickle index 3b301ff634be2cec051232ba5da05a3ec019bf2d..a09808c748731b4f2978ff24db6cb65fa08dc137 100644 GIT binary patch delta 94 zcmaFK)WOWsz%sRVBFjX9+9?_ttlkVgY`KXgMVS>-CQc7#6rK1w+@gm+IX|Z~H!r@p tG%YQ&BDHu*>=cb2=4iv%DL~RFc1mh#QgIJ=UTJPTL}Bq1Z delta 63 zcmeBRe#ykrz%n&%BFjW3Z-$9WgBgV<{tQ=-oubjh9Bmjo1xOmjPDyQ_l2qKoomZM0 SpPZjlnwwWV#haxxNe=+N*A~D4 diff --git a/pickle_testcases/0.5/transform_center.pickle b/pickle_testcases/0.5/transform_center.pickle index f98f8d26057ba5e397af150337c1f414b72e1874..369a585eb2a26181f807a788eccd5374652093ab 100644 GIT binary patch delta 55 zcmcb`_?VHUfo1BAi7ZoDn9~f+CeBe&5>Pnw9|-R6;&fmDgDE9bl6qK^^Gowerg*26 HCg}kHP%ISo delta 46 zcmaFNc#Dyxfo1CXi7Zo@nbIsK&Q+1(`0*bIqNh7dDVdVg!3B~y}0lk@3B~y}0lk@p^7FGx3tTdjOQuYoqS-jbn;~|JM(vc^ gDLssdQ@k0AYo};ruzNH0FeXj$W-d)_o06ml02jF%S^xk5