From 55317355e91d8d6130636c849b4b6d46bd9d8cd5 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 01:41:24 +0530 Subject: [PATCH 01/40] Added redis, zarr, mongodb backeneds --- signac/core/mongodbcollection.py | 149 +++++++++++++++++++++++++++++++ signac/core/rediscollection.py | 142 +++++++++++++++++++++++++++++ signac/core/zarrcollection.py | 135 ++++++++++++++++++++++++++++ tests/test_synced_collection.py | 43 +++++++++ 4 files changed, 469 insertions(+) create mode 100644 signac/core/mongodbcollection.py create mode 100644 signac/core/rediscollection.py create mode 100644 signac/core/zarrcollection.py diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py new file mode 100644 index 000000000..3bac399c3 --- /dev/null +++ b/signac/core/mongodbcollection.py @@ -0,0 +1,149 @@ +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +"""Implements Mongo-backend. + +This implements the Mongo-backend for SyncedCollection API by +implementing sync and load methods. +""" +import json +import pymongo + +from .synced_collection import SyncedCollection +from .syncedattrdict import SyncedAttrDict +from .synced_list import SyncedList + + +class MongoCollection(SyncedCollection): + """Implement sync and load using a Mongo backend.""" + + backend = __name__ # type: ignore + + def __init__(self, name=None, client=None, database='signac_db', collection='collection' + mongo_kwargs=None, **kwargs): + if client is None: + mongo_kwargs = mongo_kwargs if mongo_kwargs is not None else {} + self._client = pymongo.MongoClient(**mongo_kwargs) + else: + self._client = client + self._db = self._client.get_database(database) + self._collection = self._db.get_collection(collection) + self._name = name + self._key = 'name' + if (name is None) == (self._parent is None): + raise ValueError( + "Illegal argument combination, one of the two arguments, " + "parent or name must be None, but not both.") + super().__init__(**kwargs) + + + def _load(self): + """Load the data from a Mongo-database.""" + blob = self._collection.find_one({self._key: self._name}) + return blob['data'] if blob is not None else None + + def _sync(self): + """Write the data from Mongo-database.""" + data = self.to_base() + data_to_insert = {self._key: self._name, 'data': data} + self._collection.replace_one({self._key: self._name}, data_to_insert, True) + + +class MongoDict(MongoCollection, SyncedAttrDict): + """A dict-like mapping interface to a persistent Mongo-database. + + The MongoDict inherits from :class:`~core.collection_api.MongoCollection` + and :class:`~core.syncedattrdict.SyncedAttrDict`. + + .. code-block:: python + + doc = MongoDict('data') + doc['foo'] = "bar" + assert doc.foo == doc['foo'] == "bar" + assert 'foo' in doc + del doc['foo'] + + .. code-block:: python + + >>> doc['foo'] = dict(bar=True) + >>> doc + {'foo': {'bar': True}} + >>> doc.foo.bar = False + {'foo': {'bar': False}} + + .. warning:: + + While the MongoDict object behaves like a dictionary, there are + important distinctions to remember. In particular, because operations + are reflected as changes to an underlying database, copying (even deep + copying) a MongoDict instance may exhibit unexpected behavior. If a + true copy is required, you should use the `to_base()` method to get a + dictionary representation, and if necessary construct a new MongoDict + instance: `new_dict = MongoDict(old_dict.to_base())`. + + Parameters + ---------- + name: str + The name of the collection (Default value = None). + client: + A Mongo client. + mongo_kwargs: dict + kwargs arguments passed through to the `pymongo.MongoClient` function. + database : string + Name of database (Default value = 'signac_db'). + collection : string + Name of collection (Default value = 'collection') + data: mapping, optional + The intial data pass to MOngoDict. Defaults to `dict()` + parent: object, optional + A parent instance of MongoDict or None (Default value = None). + """ + pass + + +class MongoList(MongoCollection, SyncedList): + """A non-string sequence interface to a persistent Mongo file. + + The MongoDict inherits from :class:`~core.synced_collection.SyncedCollection` + and :class:`~core.syncedlist.SyncedList`. + + .. code-block:: python + + synced_list = RedisList('data') + synced_list.append("bar") + assert synced_list[0] == "bar" + assert len(synced_list) == 1 + del synced_list[0] + + .. warning:: + + While the MongoList object behaves like a list, there are + important distinctions to remember. In particular, because operations + are reflected as changes to an underlying database, copying (even deep + copying) a MongoList instance may exhibit unexpected behavior. If a + true copy is required, you should use the `to_base()` method to get a + dictionary representation, and if necessary construct a new MongoList + instance: `new_list = MongoList(old_list.to_base())`. + + Parameters + ---------- + name: str + The name of the collection (Default value = None). + client: + A Mongo client. + Mongo_kwargs: dict + kwargs arguments passed through to the `Mongo.Mongo` function. + database : string + Name of database (Default value = 'signac_db'). + collection : string + Name of collection (Default value = 'collection') + data: mapping, optional + The intial data pass to MongoDict. Defaults to `list()` + parent: object, optional + A parent instance of MongoDict or None (Default value = None). + """ + + pass + + +SyncedCollection.register(MongoDict, MongoList) diff --git a/signac/core/rediscollection.py b/signac/core/rediscollection.py new file mode 100644 index 000000000..e2a6d3281 --- /dev/null +++ b/signac/core/rediscollection.py @@ -0,0 +1,142 @@ +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +"""Implements Redis-backend. + +This implements the Redis-backend for SyncedCollection API by +implementing sync and load methods. +""" +import json +import redis + +from .synced_collection import SyncedCollection +from .syncedattrdict import SyncedAttrDict +from .synced_list import SyncedList + + +class RedisCollection(SyncedCollection): + """Implement sync and load using a Redis backend.""" + + backend = __name__ # type: ignore + + def __init__(self, name=None, client=None, redis_kwargs=None, **kwargs): + if client is None: + redis_kwargs = redis_kwargs if redis_kwargs is not None else {} + self._client = redis.Redis(**redis_kwargs) + else: + self._client = client + self._name = name + super().__init__(**kwargs) + if (name is None) == (self._parent is None): + raise ValueError( + "Illegal argument combination, one of the two arguments, " + "parent or name must be None, but not both.") + + def _load(self): + """Load the data from a Redis-database.""" + blob = self._client.get(self._name) + return json.loads(blob) if blob is not None else blob + + def _sync(self): + """Write the data from Redis-database.""" + data = self.to_base() + # Serialize data: + blob = json.dumps(data).encode() + + self._client.set(self._name, blob) + + +class RedisDict(RedisCollection, SyncedAttrDict): + """A dict-like mapping interface to a persistent Redis-database. + + The RedisDict inherits from :class:`~core.rediscollection.RedisCollection` + and :class:`~core.syncedattrdict.SyncedAttrDict`. + + .. code-block:: python + + doc = RedisDict('data') + doc['foo'] = "bar" + assert doc.foo == doc['foo'] == "bar" + assert 'foo' in doc + del doc['foo'] + + .. code-block:: python + + >>> doc['foo'] = dict(bar=True) + >>> doc + {'foo': {'bar': True}} + >>> doc.foo.bar = False + {'foo': {'bar': False}} + + .. warning:: + + While the RedisDict object behaves like a dictionary, there are + important distinctions to remember. In particular, because operations + are reflected as changes to an underlying database, copying (even deep + copying) a RedisDict instance may exhibit unexpected behavior. If a + true copy is required, you should use the `to_base()` method to get a + dictionary representation, and if necessary construct a new RedisDict + instance: `new_dict = RedisDict(old_dict.to_base())`. + + Parameters + ---------- + name: str + The name of the collection (Default value = None). + client: + A redis client. + redis_kwargs: dict + kwargs arguments passed through to the `redis.Redis` function. + data: mapping, optional + The intial data pass to RedisDict. Defaults to `dict()` + parent: object, optional + A parent instance of RedisDict or None (Default value = None). + + + """ + + pass + + +class RedisList(RedisCollection, SyncedList): + """A non-string sequence interface to a persistent Redis file. + + The RedisDict inherits from :class:`~core.collection_api.SyncedCollection` + and :class:`~core.syncedlist.SyncedList`. + + .. code-block:: python + + synced_list = RedisList('data') + synced_list.append("bar") + assert synced_list[0] == "bar" + assert len(synced_list) == 1 + del synced_list[0] + + .. warning:: + + While the RedisList object behaves like a list, there are + important distinctions to remember. In particular, because operations + are reflected as changes to an underlying database, copying (even deep + copying) a RedisList instance may exhibit unexpected behavior. If a + true copy is required, you should use the `to_base()` method to get a + dictionary representation, and if necessary construct a new RedisList + instance: `new_list = RedisList(old_list.to_base())`. + + Parameters + ---------- + name: str + The name of the collection (Default value = None). + client: + A redis client. + redis_kwargs: dict + kwargs arguments passed through to the `redis.Redis` function. + data: mapping, optional + The intial data pass to RedisDict. Defaults to `list()` + parent: object, optional + A parent instance of RedisDict or None (Default value = None). + + """ + + pass + + +SyncedCollection.register(RedisDict, RedisList) diff --git a/signac/core/zarrcollection.py b/signac/core/zarrcollection.py new file mode 100644 index 000000000..3902fd81b --- /dev/null +++ b/signac/core/zarrcollection.py @@ -0,0 +1,135 @@ +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +"""Implements Zarr-backend. + +This implements the Zarr-backend for SyncedCollection API by +implementing sync and load methods. +""" +import zarr +import numcodecs + +from .synced_collection import SyncedCollection +from .syncedattrdict import SyncedAttrDict +from .synced_list import SyncedList + + +class ZarrCollection(SyncedCollection): + """Implement sync and load using a Zarr backend.""" + + backend = __name__ # type: ignore + + def __init__(self, name=None, store=None, **kwargs): + self._root = zarr.group(store=store) + self._name = name + super().__init__(**kwargs) + if (name is None) == (self._parent is None): + raise ValueError( + "Illegal argument combination, one of the two arguments, " + "parent or name must be None, but not both.") + + def _load(self): + """Load the data from a Radis-database.""" + try: + dataset = self._root[self._name] + data = dataset[0] + except KeyError: + data = None + return data + + def _sync(self): + """Write the data from Radis-database.""" + data = self.to_base() + # Serialize data: + dataset = self._root.require_dataset( + self._name, overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) + dataset[0] = data + + +class ZarrDict(ZarrCollection, SyncedAttrDict): + """A dict-like mapping interface to a persistent Zarr-database. + + The ZarrDict inherits from :class:`~core.collection_api.ZarrCollection` + and :class:`~core.syncedattrdict.SyncedAttrDict`. + + .. code-block:: python + + doc = ZarrDict('data') + doc['foo'] = "bar" + assert doc.foo == doc['foo'] == "bar" + assert 'foo' in doc + del doc['foo'] + + .. code-block:: python + + >>> doc['foo'] = dict(bar=True) + >>> doc + {'foo': {'bar': True}} + >>> doc.foo.bar = False + {'foo': {'bar': False}} + + .. warning:: + + While the ZarrDict object behaves like a dictionary, there are + important distinctions to remember. In particular, because operations + are reflected as changes to an underlying database, copying (even deep + copying) a ZarrDict instance may exhibit unexpected behavior. If a + true copy is required, you should use the `to_base()` method to get a + dictionary representation, and if necessary construct a new ZarrDict + instance: `new_dict = ZarrDict(old_dict.to_base())`. + + Parameters + ---------- + name: str, optional + The name of the collection (Default value = None). + data: mapping, optional + The intial data pass to ZarrDict. Defaults to `dict()`. + store: mapping + A zarr store to synchronise the data + parent: object, optional + A parent instance of ZarrDict or None (Default value = None). + """ + + pass + + +class ZarrList(ZarrCollection, SyncedList): + """A non-string sequence interface to a persistent Zarr file. + + The ZarrDict inherits from :class:`~core.collection_api.ZarrCollection` + and :class:`~core.syncedlist.SyncedList`. + + .. code-block:: python + + synced_list = ZarrList('data') + synced_list.append("bar") + assert synced_list[0] == "bar" + assert len(synced_list) == 1 + del synced_list[0] + + .. warning:: + + While the ZarrList object behaves like a list, there are + important distinctions to remember. In particular, because operations + are reflected as changes to an underlying database, copying (even deep + copying) a ZarrList instance may exhibit unexpected behavior. If a + true copy is required, you should use the `to_base()` method to get a + dictionary representation, and if necessary construct a new ZarrList + instance: `new_list = ZarrList(old_list.to_base())`. + + Parameters + ---------- + name: str + The name of the collection. + data: mapping, optional + The intial data pass to ZarrDict. Defaults to `list()`. + store: mapping + A zarr store to synchronise the data + parent: object, optional + A parent instance of ZarrDict or None (Default value = None). + """ + + pass + + +SyncedCollection.register(ZarrDict, ZarrList) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 0792be926..65dda35cd 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -5,6 +5,7 @@ import uuid import os import json +import numcodecs from tempfile import TemporaryDirectory from collections.abc import MutableMapping from collections.abc import MutableSequence @@ -13,6 +14,8 @@ from signac.core.synced_list import SyncedCollection from signac.core.jsoncollection import JSONDict from signac.core.jsoncollection import JSONList +from signac.core.zarrcollection import ZarrDict +from signac.core.zarrcollection import ZarrList from signac.errors import InvalidKeyError from signac.errors import KeyTypeError @@ -22,6 +25,12 @@ except ImportError: NUMPY = False +try: + import zarr + Zarr = True +except ImportError: + Zarr = False + FN_JSON = 'test.json' @@ -602,3 +611,37 @@ def synced_list(self): self._backend_kwargs = {'filename': self._fn_, 'write_concern': True} yield JSONList(**self._backend_kwargs) self._tmp_dir.cleanup() + + +@pytest.mark.skipif(not Zarr, reason='test requires the zarr package') +class TestZarrDict(TestJSONDict): + + @pytest.fixture(autouse=True) + def synced_dict(self): + self._tmp_dir = TemporaryDirectory(prefix='jsondict_') + self._store = zarr.DirectoryStore(self._tmp_dir.name) + self._backend_kwargs = {'name': 'test', 'store': self._store} + yield ZarrDict(**self._backend_kwargs) + self._tmp_dir.cleanup() + + def store(self, data): + dataset = zarr.group(self._store).require_dataset( + 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) + dataset[0] = data + + +@pytest.mark.skipif(not Zarr, reason='test requires the zarr package') +class TestZarrList(TestJSONList): + + @pytest.fixture(autouse=True) + def synced_list(self): + self._tmp_dir = TemporaryDirectory(prefix='jsondict_') + self._store = zarr.DirectoryStore(self._tmp_dir.name) + self._backend_kwargs = {'name': 'test', 'store': self._store} + yield ZarrList(**self._backend_kwargs) + self._tmp_dir.cleanup() + + def store(self, data): + dataset = zarr.group(self._store).require_dataset( + 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) + dataset[0] = data From 22c4235e040c56055500bba0077c1a1cd45f48ad Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 01:43:23 +0530 Subject: [PATCH 02/40] Added redis, zarr, mongodb backeneds --- signac/core/mongodbcollection.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index 3bac399c3..f2c797f68 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -19,7 +19,7 @@ class MongoCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, client=None, database='signac_db', collection='collection' + def __init__(self, name=None, client=None, database='signac_db', collection='collection', mongo_kwargs=None, **kwargs): if client is None: mongo_kwargs = mongo_kwargs if mongo_kwargs is not None else {} @@ -36,7 +36,6 @@ def __init__(self, name=None, client=None, database='signac_db', collection='col "parent or name must be None, but not both.") super().__init__(**kwargs) - def _load(self): """Load the data from a Mongo-database.""" blob = self._collection.find_one({self._key: self._name}) @@ -91,7 +90,7 @@ class MongoDict(MongoCollection, SyncedAttrDict): kwargs arguments passed through to the `pymongo.MongoClient` function. database : string Name of database (Default value = 'signac_db'). - collection : string + collection : string Name of collection (Default value = 'collection') data: mapping, optional The intial data pass to MOngoDict. Defaults to `dict()` @@ -135,7 +134,7 @@ class MongoList(MongoCollection, SyncedList): kwargs arguments passed through to the `Mongo.Mongo` function. database : string Name of database (Default value = 'signac_db'). - collection : string + collection : string Name of collection (Default value = 'collection') data: mapping, optional The intial data pass to MongoDict. Defaults to `list()` From 80ef22dda559daf83d09ad8cc0d6b20d8be16d7c Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 01:44:35 +0530 Subject: [PATCH 03/40] Removed extra import --- signac/core/mongodbcollection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index f2c797f68..f72b3da96 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -6,7 +6,6 @@ This implements the Mongo-backend for SyncedCollection API by implementing sync and load methods. """ -import json import pymongo from .synced_collection import SyncedCollection From 0203f28fd680c31b8e173e9f4c20ad918e3096e7 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 01:56:24 +0530 Subject: [PATCH 04/40] Added zarr, redis to requirement-dev.txt --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1a3af8ef1..b506f78f1 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,5 @@ pytest-subtests==0.3.1 pydocstyle==5.0.2 pytest-cov==2.10.0 pymongo==3.10.1 +zarr==2.4.0 +redis 3.5.3 From 113b27401fb4dc884a92b7255464ced2b930a001 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 01:57:34 +0530 Subject: [PATCH 05/40] Correction requirement-dev.txt --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b506f78f1..affb371f9 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -12,4 +12,4 @@ pydocstyle==5.0.2 pytest-cov==2.10.0 pymongo==3.10.1 zarr==2.4.0 -redis 3.5.3 +redis==3.5.3 From b1795f6bfdf86095d74fef8d7a90890eb3d04196 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 02:06:06 +0530 Subject: [PATCH 06/40] moved numcodecs import in tests --- signac/core/mongodbcollection.py | 1 - tests/test_synced_collection.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index f72b3da96..1f5ce8f54 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -6,7 +6,6 @@ This implements the Mongo-backend for SyncedCollection API by implementing sync and load methods. """ -import pymongo from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 65dda35cd..dfccc238e 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -5,7 +5,6 @@ import uuid import os import json -import numcodecs from tempfile import TemporaryDirectory from collections.abc import MutableMapping from collections.abc import MutableSequence @@ -27,6 +26,7 @@ try: import zarr + import numcodecs # zarr depends on numcodecs Zarr = True except ImportError: Zarr = False From 84ec0b51381c87684f4501a4173af038339560e2 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 1 Aug 2020 02:07:33 +0530 Subject: [PATCH 07/40] Linting error --- signac/core/mongodbcollection.py | 1 + tests/test_synced_collection.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index 1f5ce8f54..f72b3da96 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -6,6 +6,7 @@ This implements the Mongo-backend for SyncedCollection API by implementing sync and load methods. """ +import pymongo from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index dfccc238e..bde125884 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -26,7 +26,7 @@ try: import zarr - import numcodecs # zarr depends on numcodecs + import numcodecs # zarr depends on numcodecs Zarr = True except ImportError: Zarr = False From 8c0d31ba33fd7a3628ec12693385fc88a6747479 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 3 Aug 2020 02:32:04 +0530 Subject: [PATCH 08/40] Renamed zarr temp dir --- tests/test_synced_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index bde125884..97354a6b1 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -635,7 +635,7 @@ class TestZarrList(TestJSONList): @pytest.fixture(autouse=True) def synced_list(self): - self._tmp_dir = TemporaryDirectory(prefix='jsondict_') + self._tmp_dir = TemporaryDirectory(prefix='zarrdict_') self._store = zarr.DirectoryStore(self._tmp_dir.name) self._backend_kwargs = {'name': 'test', 'store': self._store} yield ZarrList(**self._backend_kwargs) From 83492a4b84a27f543395af1b30f9695babbb7067 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2020 01:37:41 +0530 Subject: [PATCH 09/40] Added test for redis --- tests/test_synced_collection.py | 56 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 97354a6b1..6882b975b 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -15,6 +15,8 @@ from signac.core.jsoncollection import JSONList from signac.core.zarrcollection import ZarrDict from signac.core.zarrcollection import ZarrList +from signac.core.rediscollection import RedisDict +from signac.core.rediscollection import RedisList from signac.errors import InvalidKeyError from signac.errors import KeyTypeError @@ -27,9 +29,23 @@ try: import zarr import numcodecs # zarr depends on numcodecs - Zarr = True + ZARR = True except ImportError: - Zarr = False + ZARR = False + +try: + import redis + try: + RedisClient = redis.Redis() + test_key = str(uuid.uuid4()) + RedisClient.set(test_key, 0) + assert json.loads(RedisClient.get(test_key)) == 0 + RedisClient.flushall() + REDIS = True + except (redis.exceptions.ConnectionError, AssertionError) : + REDIS = False +except ImportError: + REDIS = False FN_JSON = 'test.json' @@ -630,18 +646,46 @@ def store(self, data): dataset[0] = data -@pytest.mark.skipif(not Zarr, reason='test requires the zarr package') +@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') class TestZarrList(TestJSONList): @pytest.fixture(autouse=True) def synced_list(self): - self._tmp_dir = TemporaryDirectory(prefix='zarrdict_') + self._tmp_dir = TemporaryDirectory(prefix='zarrlist_') self._store = zarr.DirectoryStore(self._tmp_dir.name) - self._backend_kwargs = {'name': 'test', 'store': self._store} - yield ZarrList(**self._backend_kwargs) + self._name = 'test' + yield ZarrList(name=self._name, store=self._store) self._tmp_dir.cleanup() def store(self, data): dataset = zarr.group(self._store).require_dataset( 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) dataset[0] = data + + +@pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') +class TestRedisDict(TestJSONDict): + + @pytest.fixture + def synced_dict(self, request): + self._client = RedisClient + request.addfinalizer(self._client.flushall) + self._name ='test' + yield RedisDict(name=self._name, client=self._client) + + def store(self, data): + self._client.set(self._name, json.dumps(data).encode()) + + +@pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') +class TestRedisList(TestJSONList): + + @pytest.fixture + def synced_list(self, request): + self._client = RedisClient + request.addfinalizer(self._client.flushall) + self._name ='test' + yield RedisList(name=self._name, client=self._client) + + def store(self, data): + self._client.set(self._name, json.dumps(data).encode()) From 456878f93af1a366b15d2de87833baef5fe9552e Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2020 01:41:15 +0530 Subject: [PATCH 10/40] Added test for redis --- tests/test_synced_collection.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 6882b975b..2414aff68 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -29,9 +29,9 @@ try: import zarr import numcodecs # zarr depends on numcodecs - ZARR = True + Zarr = True except ImportError: - ZARR = False + Zarr = False try: import redis @@ -42,7 +42,7 @@ assert json.loads(RedisClient.get(test_key)) == 0 RedisClient.flushall() REDIS = True - except (redis.exceptions.ConnectionError, AssertionError) : + except (redis.exceptions.ConnectionError, AssertionError): REDIS = False except ImportError: REDIS = False @@ -670,7 +670,7 @@ class TestRedisDict(TestJSONDict): def synced_dict(self, request): self._client = RedisClient request.addfinalizer(self._client.flushall) - self._name ='test' + self._name = 'test' yield RedisDict(name=self._name, client=self._client) def store(self, data): @@ -684,7 +684,7 @@ class TestRedisList(TestJSONList): def synced_list(self, request): self._client = RedisClient request.addfinalizer(self._client.flushall) - self._name ='test' + self._name = 'test' yield RedisList(name=self._name, client=self._client) def store(self, data): From 5275f3acfcbccf7bf5090cab8fbca10106291efa Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2020 01:42:47 +0530 Subject: [PATCH 11/40] Added test for redis-backend --- tests/test_synced_collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 2414aff68..14d372188 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -646,7 +646,7 @@ def store(self, data): dataset[0] = data -@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') +@pytest.mark.skipif(not Zarr, reason='test requires the zarr package') class TestZarrList(TestJSONList): @pytest.fixture(autouse=True) From 0c8cc25239f6d7dc0edfe01d7efb9ace50d3526e Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2020 01:46:52 +0530 Subject: [PATCH 12/40] Added test for redis backend and changed Zarr to ZARR --- tests/test_synced_collection.py | 35 ++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 14d372188..cb22b5cea 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -8,7 +8,7 @@ from tempfile import TemporaryDirectory from collections.abc import MutableMapping from collections.abc import MutableSequence -from copy import deepcopy +from copy import copy from signac.core.synced_list import SyncedCollection from signac.core.jsoncollection import JSONDict @@ -60,22 +60,29 @@ class TestSyncedCollectionBase: # this fixture sets temprary directory for tests @pytest.fixture(autouse=True) def synced_collection(self): - self._tmp_dir = TemporaryDirectory(prefix='jsondict_') + self._tmp_dir = TemporaryDirectory(prefix='synced_collection_') self._fn_ = os.path.join(self._tmp_dir.name, FN_JSON) yield self._tmp_dir.cleanup() - def test_from_base(self): + def test_from_base_json(self): sd = SyncedCollection.from_base(filename=self._fn_, data={'a': 0}, backend='signac.core.jsoncollection') assert isinstance(sd, JSONDict) assert 'a' in sd assert sd['a'] == 0 - # invalid input - with pytest.raises(ValueError): - SyncedCollection.from_base(data={'a': 0}, filename=self._fn_) + @pytest.mark.skipif(not ZARR, reason='test requires the zarr package') + def test_from_base_zarr(self): + sd = SyncedCollection.from_base(name='test', + data={'a': 0}, backend='signac.core.zarrcollection') + assert isinstance(sd, ZarrDict) + assert 'a' in sd + assert sd['a'] == 0 + def test_from_base_no_backend(self): + with pytest.raises(ValueError): + SyncedCollection.from_base(data={'a': 0}) class TestJSONDict: @@ -304,7 +311,7 @@ def test_call(self, synced_dict, testdata): def test_reopen(self, synced_dict, testdata): key = 'reopen' synced_dict[key] = testdata - synced_dict2 = deepcopy(synced_dict) + synced_dict2 = copy(synced_dict) synced_dict.sync() del synced_dict # possibly unsafe synced_dict2.load() @@ -399,7 +406,7 @@ class TestJSONList: @pytest.fixture def synced_list(self): - self._tmp_dir = TemporaryDirectory(prefix='jsondict_') + self._tmp_dir = TemporaryDirectory(prefix='jsonlist_') self._fn_ = os.path.join(self._tmp_dir.name, FN_JSON) self._backend_kwargs = {'filename': self._fn_, 'write_concern': self._write_concern} yield JSONList(**self._backend_kwargs) @@ -545,14 +552,14 @@ def test_update_recursive(self, synced_list): self.store(data1) assert synced_list == data1 - # inavlid data in file + # invalid data in file data2 = {'a': 1} self.store(data2) with pytest.raises(ValueError): synced_list.load() def test_reopen(self, synced_list, testdata): - synced_list2 = deepcopy(synced_list) + synced_list2 = copy(synced_list) synced_list.append(testdata) synced_list.sync() del synced_list # possibly unsafe @@ -629,15 +636,15 @@ def synced_list(self): self._tmp_dir.cleanup() -@pytest.mark.skipif(not Zarr, reason='test requires the zarr package') +@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') class TestZarrDict(TestJSONDict): @pytest.fixture(autouse=True) def synced_dict(self): - self._tmp_dir = TemporaryDirectory(prefix='jsondict_') + self._tmp_dir = TemporaryDirectory(prefix='zarrdict_') self._store = zarr.DirectoryStore(self._tmp_dir.name) - self._backend_kwargs = {'name': 'test', 'store': self._store} - yield ZarrDict(**self._backend_kwargs) + self._name = 'test' + yield ZarrDict(name=self._name, store=self._store) self._tmp_dir.cleanup() def store(self, data): From dc7f8aedc8be8cd04e3ff207cd20f6ba5483471d Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2020 01:48:08 +0530 Subject: [PATCH 13/40] Linting changes --- tests/test_synced_collection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index cb22b5cea..d9a80aae1 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -29,9 +29,9 @@ try: import zarr import numcodecs # zarr depends on numcodecs - Zarr = True + ZARR = True except ImportError: - Zarr = False + ZARR = False try: import redis @@ -653,7 +653,7 @@ def store(self, data): dataset[0] = data -@pytest.mark.skipif(not Zarr, reason='test requires the zarr package') +@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') class TestZarrList(TestJSONList): @pytest.fixture(autouse=True) @@ -672,7 +672,7 @@ def store(self, data): @pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') class TestRedisDict(TestJSONDict): - + @pytest.fixture def synced_dict(self, request): self._client = RedisClient From dd2e4fa5ef4d4884921d9291b317e1107f7ad5d1 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 5 Aug 2020 01:48:58 +0530 Subject: [PATCH 14/40] Linting changes --- tests/test_synced_collection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index d9a80aae1..5fcb33746 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -84,6 +84,7 @@ def test_from_base_no_backend(self): with pytest.raises(ValueError): SyncedCollection.from_base(data={'a': 0}) + class TestJSONDict: @pytest.fixture From 5d765b4eed0909559cb403ff59608ad3c2c827f1 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 6 Aug 2020 23:25:42 +0530 Subject: [PATCH 15/40] Added test for mongocollection --- signac/core/mongodbcollection.py | 5 ++- tests/test_synced_collection.py | 76 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index f72b3da96..6a4cc8f28 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -28,12 +28,13 @@ def __init__(self, name=None, client=None, database='signac_db', collection='col self._db = self._client.get_database(database) self._collection = self._db.get_collection(collection) self._name = name - self._key = 'name' + self._key = type(self).__name__ + '::name' + super().__init__(**kwargs) if (name is None) == (self._parent is None): raise ValueError( "Illegal argument combination, one of the two arguments, " "parent or name must be None, but not both.") - super().__init__(**kwargs) + def _load(self): """Load the data from a Mongo-database.""" diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 5fcb33746..9145be9c8 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -17,6 +17,8 @@ from signac.core.zarrcollection import ZarrList from signac.core.rediscollection import RedisDict from signac.core.rediscollection import RedisList +from signac.core.mongodbcollection import MongoDict +from signac.core.mongodbcollection import MongoList from signac.errors import InvalidKeyError from signac.errors import KeyTypeError @@ -47,6 +49,14 @@ except ImportError: REDIS = False +try: + import pymongo + import bson # required for invalid data error + MongoClient = pymongo.MongoClient() + PYMONGO = True +except ImportError: + PYMONGO = False + FN_JSON = 'test.json' @@ -80,6 +90,23 @@ def test_from_base_zarr(self): assert 'a' in sd assert sd['a'] == 0 + @pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package.') + def test_from_base_mongo(self): + sd = SyncedCollection.from_base(name='test', data={'a': 0}, client=MongoClient, + backend='signac.core.mongodbcollection') + assert isinstance(sd, MongoDict) + assert 'a' in sd + assert sd['a'] == 0 + + @pytest.mark.skipif(not REDIS, reason='test requires the redis package ' + 'and running redis-server') + def test_from_base_redis(self): + sd = SyncedCollection.from_base(name='test', data={'a': 0}, client=RedisClient, + backend='signac.core.rediscollection') + assert isinstance(sd, ResisDict) + assert 'a' in sd + assert sd['a'] == 0 + def test_from_base_no_backend(self): with pytest.raises(ValueError): SyncedCollection.from_base(data={'a': 0}) @@ -697,3 +724,52 @@ def synced_list(self, request): def store(self, data): self._client.set(self._name, json.dumps(data).encode()) + + +@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package') +class TestMongoDict(TestJSONDict): + + @pytest.fixture + def synced_dict(self, request): + self._client = MongoClient + self._name = 'test' + yield MongoDict(name=self._name, client=self._client, + database='test_db', collection='test_dict') + self._client.test_db.test_dict.drop() + + def store(self, data): + data_to_insert = {'MongoDict::name': self._name, 'data': data} + self._client.test_db.test_dict.replace_one({'MongoDict::name': self._name}, data_to_insert) + + + def test_write_invalid_type(self, synced_dict, testdata): + # mongodict return InvalidDocument error for objects + class Foo(object): + pass + + key = 'write_invalid_type' + synced_dict[key] = testdata + assert len(synced_dict) == 1 + assert synced_dict[key] == testdata + d2 = Foo() + with pytest.raises(bson.errors.InvalidDocument): + synced_dict[key + '2'] = d2 + assert len(synced_dict) == 1 + assert synced_dict[key] == testdata + + + +@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package') +class TestMongoList(TestJSONList): + + @pytest.fixture + def synced_list(self, request): + self._client = MongoClient + self._name = 'test' + yield MongoList(name=self._name, client=self._client, + database='test_db', collection='test_list') + self._client.test_db.test_list.drop() + + def store(self, data): + data_to_insert = {'MongoList::name': self._name, 'data': data} + self._client.test_db.test_list.replace_one({'MongoList::name': self._name}, data_to_insert) \ No newline at end of file From 5dd9e154d4a5fe65fdb6feb2a6fc910f98f08465 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 6 Aug 2020 23:28:25 +0530 Subject: [PATCH 16/40] Linting Changes --- signac/core/mongodbcollection.py | 1 - tests/test_synced_collection.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index 6a4cc8f28..041440c90 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -35,7 +35,6 @@ def __init__(self, name=None, client=None, database='signac_db', collection='col "Illegal argument combination, one of the two arguments, " "parent or name must be None, but not both.") - def _load(self): """Load the data from a Mongo-database.""" blob = self._collection.find_one({self._key: self._name}) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 9145be9c8..cc37c3abb 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -103,7 +103,7 @@ def test_from_base_mongo(self): def test_from_base_redis(self): sd = SyncedCollection.from_base(name='test', data={'a': 0}, client=RedisClient, backend='signac.core.rediscollection') - assert isinstance(sd, ResisDict) + assert isinstance(sd, RedisDict) assert 'a' in sd assert sd['a'] == 0 @@ -741,7 +741,6 @@ def store(self, data): data_to_insert = {'MongoDict::name': self._name, 'data': data} self._client.test_db.test_dict.replace_one({'MongoDict::name': self._name}, data_to_insert) - def test_write_invalid_type(self, synced_dict, testdata): # mongodict return InvalidDocument error for objects class Foo(object): @@ -758,7 +757,6 @@ class Foo(object): assert synced_dict[key] == testdata - @pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package') class TestMongoList(TestJSONList): From c1ded73d027b845524fba12baffece4435f18619 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 6 Aug 2020 23:29:41 +0530 Subject: [PATCH 17/40] Added comments --- tests/test_synced_collection.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index cc37c3abb..a075d566e 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -38,11 +38,12 @@ try: import redis try: + # try to connect to server RedisClient = redis.Redis() test_key = str(uuid.uuid4()) RedisClient.set(test_key, 0) - assert json.loads(RedisClient.get(test_key)) == 0 - RedisClient.flushall() + assert RedisClient.get(test_key) == b'0' # redis store data as bytes + RedisClient.delete(test_key) REDIS = True except (redis.exceptions.ConnectionError, AssertionError): REDIS = False @@ -770,4 +771,4 @@ def synced_list(self, request): def store(self, data): data_to_insert = {'MongoList::name': self._name, 'data': data} - self._client.test_db.test_list.replace_one({'MongoList::name': self._name}, data_to_insert) \ No newline at end of file + self._client.test_db.test_list.replace_one({'MongoList::name': self._name}, data_to_insert) From 1a1b0751e6880aa9fe03cf7255097cfdaae0c33b Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 7 Aug 2020 01:01:42 +0530 Subject: [PATCH 18/40] Added test for mongodb server --- tests/test_synced_collection.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index a075d566e..f6a71b72f 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -53,8 +53,17 @@ try: import pymongo import bson # required for invalid data error - MongoClient = pymongo.MongoClient() - PYMONGO = True + try: + # test the mongodb server + MongoClient = pymongo.MongoClient() + tmp_collection = MongoClient['test_db']['test'] + tmp_collection.insert_one({'test': '0'}) + ret = tmp_collection.find_one({'test': '0'}) + assert ret['test'] == '0' + tmp_collection.drop() + PYMONGO = True + except (pymongo.errors.ServerSelectionTimeoutError, AssertionError): + PYMONGO = False except ImportError: PYMONGO = False @@ -727,7 +736,7 @@ def store(self, data): self._client.set(self._name, json.dumps(data).encode()) -@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package') +@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') class TestMongoDict(TestJSONDict): @pytest.fixture @@ -758,7 +767,7 @@ class Foo(object): assert synced_dict[key] == testdata -@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package') +@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') class TestMongoList(TestJSONList): @pytest.fixture From c17abb8bcd31ca16bf963884132daec7c12769a3 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 7 Aug 2020 01:04:17 +0530 Subject: [PATCH 19/40] Moved import to class __init__ --- signac/core/mongodbcollection.py | 4 ++-- signac/core/rediscollection.py | 3 ++- signac/core/zarrcollection.py | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index 041440c90..62315a438 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -6,8 +6,6 @@ This implements the Mongo-backend for SyncedCollection API by implementing sync and load methods. """ -import pymongo - from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict from .synced_list import SyncedList @@ -20,6 +18,8 @@ class MongoCollection(SyncedCollection): def __init__(self, name=None, client=None, database='signac_db', collection='collection', mongo_kwargs=None, **kwargs): + import pymongo + if client is None: mongo_kwargs = mongo_kwargs if mongo_kwargs is not None else {} self._client = pymongo.MongoClient(**mongo_kwargs) diff --git a/signac/core/rediscollection.py b/signac/core/rediscollection.py index e2a6d3281..647822d88 100644 --- a/signac/core/rediscollection.py +++ b/signac/core/rediscollection.py @@ -7,7 +7,6 @@ implementing sync and load methods. """ import json -import redis from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict @@ -20,6 +19,8 @@ class RedisCollection(SyncedCollection): backend = __name__ # type: ignore def __init__(self, name=None, client=None, redis_kwargs=None, **kwargs): + import redis + if client is None: redis_kwargs = redis_kwargs if redis_kwargs is not None else {} self._client = redis.Redis(**redis_kwargs) diff --git a/signac/core/zarrcollection.py b/signac/core/zarrcollection.py index 3902fd81b..ad74c40ea 100644 --- a/signac/core/zarrcollection.py +++ b/signac/core/zarrcollection.py @@ -6,9 +6,6 @@ This implements the Zarr-backend for SyncedCollection API by implementing sync and load methods. """ -import zarr -import numcodecs - from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict from .synced_list import SyncedList @@ -20,6 +17,9 @@ class ZarrCollection(SyncedCollection): backend = __name__ # type: ignore def __init__(self, name=None, store=None, **kwargs): + import zarr + import numcodecs + self._root = zarr.group(store=store) self._name = name super().__init__(**kwargs) From 8632cdbc3c614d3fe9b04ff3a734ebbf6f6c846d Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 7 Aug 2020 01:10:34 +0530 Subject: [PATCH 20/40] removed numcodec import error --- signac/core/zarrcollection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/signac/core/zarrcollection.py b/signac/core/zarrcollection.py index ad74c40ea..15af4c937 100644 --- a/signac/core/zarrcollection.py +++ b/signac/core/zarrcollection.py @@ -18,10 +18,11 @@ class ZarrCollection(SyncedCollection): def __init__(self, name=None, store=None, **kwargs): import zarr - import numcodecs + import numcodecs # zarr depends on numcodecs self._root = zarr.group(store=store) self._name = name + self._object_codec = numcodecs.JSON() super().__init__(**kwargs) if (name is None) == (self._parent is None): raise ValueError( @@ -42,7 +43,7 @@ def _sync(self): data = self.to_base() # Serialize data: dataset = self._root.require_dataset( - self._name, overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) + self._name, overwrite=True, shape=1, dtype='object', object_codec=self._object_codec) dataset[0] = data From 6cdb6f4b5e39df35c6e788a622e4244991e02b6f Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 7 Aug 2020 01:30:21 +0530 Subject: [PATCH 21/40] Updated documentation --- signac/core/mongodbcollection.py | 12 ++++++------ signac/core/rediscollection.py | 10 ++++------ signac/core/zarrcollection.py | 4 ++-- tests/test_synced_collection.py | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/signac/core/mongodbcollection.py b/signac/core/mongodbcollection.py index 62315a438..3c0a926da 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/mongodbcollection.py @@ -83,7 +83,7 @@ class MongoDict(MongoCollection, SyncedAttrDict): ---------- name: str The name of the collection (Default value = None). - client: + client: object A Mongo client. mongo_kwargs: dict kwargs arguments passed through to the `pymongo.MongoClient` function. @@ -92,7 +92,7 @@ class MongoDict(MongoCollection, SyncedAttrDict): collection : string Name of collection (Default value = 'collection') data: mapping, optional - The intial data pass to MOngoDict. Defaults to `dict()` + The intial data pass to MongoDict. Defaults to `dict()` parent: object, optional A parent instance of MongoDict or None (Default value = None). """ @@ -107,7 +107,7 @@ class MongoList(MongoCollection, SyncedList): .. code-block:: python - synced_list = RedisList('data') + synced_list = MongoList('data') synced_list.append("bar") assert synced_list[0] == "bar" assert len(synced_list) == 1 @@ -130,15 +130,15 @@ class MongoList(MongoCollection, SyncedList): client: A Mongo client. Mongo_kwargs: dict - kwargs arguments passed through to the `Mongo.Mongo` function. + kwargs arguments passed through to the `pymongo.MongoClient` function. database : string Name of database (Default value = 'signac_db'). collection : string Name of collection (Default value = 'collection') data: mapping, optional - The intial data pass to MongoDict. Defaults to `list()` + The intial data pass to MongoList. Defaults to `list()` parent: object, optional - A parent instance of MongoDict or None (Default value = None). + A parent instance of MongoList or None (Default value = None). """ pass diff --git a/signac/core/rediscollection.py b/signac/core/rediscollection.py index 647822d88..7cc387382 100644 --- a/signac/core/rediscollection.py +++ b/signac/core/rediscollection.py @@ -83,7 +83,7 @@ class RedisDict(RedisCollection, SyncedAttrDict): ---------- name: str The name of the collection (Default value = None). - client: + client: object A redis client. redis_kwargs: dict kwargs arguments passed through to the `redis.Redis` function. @@ -91,8 +91,6 @@ class RedisDict(RedisCollection, SyncedAttrDict): The intial data pass to RedisDict. Defaults to `dict()` parent: object, optional A parent instance of RedisDict or None (Default value = None). - - """ pass @@ -126,14 +124,14 @@ class RedisList(RedisCollection, SyncedList): ---------- name: str The name of the collection (Default value = None). - client: + client: object A redis client. redis_kwargs: dict kwargs arguments passed through to the `redis.Redis` function. data: mapping, optional - The intial data pass to RedisDict. Defaults to `list()` + The intial data pass to RedisList. Defaults to `list()` parent: object, optional - A parent instance of RedisDict or None (Default value = None). + A parent instance of RedisList or None (Default value = None). """ diff --git a/signac/core/zarrcollection.py b/signac/core/zarrcollection.py index 15af4c937..f835a45c2 100644 --- a/signac/core/zarrcollection.py +++ b/signac/core/zarrcollection.py @@ -123,11 +123,11 @@ class ZarrList(ZarrCollection, SyncedList): name: str The name of the collection. data: mapping, optional - The intial data pass to ZarrDict. Defaults to `list()`. + The intial data pass to ZarrList. Defaults to `list()`. store: mapping A zarr store to synchronise the data parent: object, optional - A parent instance of ZarrDict or None (Default value = None). + A parent instance of ZarrList or None (Default value = None). """ pass diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index f6a71b72f..8ae494b77 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -52,7 +52,7 @@ try: import pymongo - import bson # required for invalid data error + import bson # required for invalid data error in test_write_invalid_type try: # test the mongodb server MongoClient = pymongo.MongoClient() From f1134523fb587a1aafc259d5405fe37b9fb06410 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 12 Aug 2020 01:59:07 +0530 Subject: [PATCH 22/40] Changed API as suggested --- .../{jsoncollection.py => collection_json.py} | 0 ...odbcollection.py => collection_mongodb.py} | 42 +++++-------------- ...rediscollection.py => collection_redis.py} | 22 ++-------- .../{zarrcollection.py => collection_zarr.py} | 20 ++++----- 4 files changed, 23 insertions(+), 61 deletions(-) rename signac/core/{jsoncollection.py => collection_json.py} (100%) rename signac/core/{mongodbcollection.py => collection_mongodb.py} (75%) rename signac/core/{rediscollection.py => collection_redis.py} (84%) rename signac/core/{zarrcollection.py => collection_zarr.py} (91%) diff --git a/signac/core/jsoncollection.py b/signac/core/collection_json.py similarity index 100% rename from signac/core/jsoncollection.py rename to signac/core/collection_json.py diff --git a/signac/core/mongodbcollection.py b/signac/core/collection_mongodb.py similarity index 75% rename from signac/core/mongodbcollection.py rename to signac/core/collection_mongodb.py index 3c0a926da..26ae88b3a 100644 --- a/signac/core/mongodbcollection.py +++ b/signac/core/collection_mongodb.py @@ -1,9 +1,9 @@ # Copyright (c) 2020 The Regents of the University of Michigan # All rights reserved. # This software is licensed under the BSD 3-Clause License. -"""Implements Mongo-backend. +"""Implements MongoDB-backend. -This implements the Mongo-backend for SyncedCollection API by +This implements the MongoDB-backend for SyncedCollection API by implementing sync and load methods. """ from .synced_collection import SyncedCollection @@ -11,22 +11,13 @@ from .synced_list import SyncedList -class MongoCollection(SyncedCollection): - """Implement sync and load using a Mongo backend.""" +class MongoDBCollection(SyncedCollection): + """Implement sync and load using a MongoDB backend.""" backend = __name__ # type: ignore - def __init__(self, name=None, client=None, database='signac_db', collection='collection', - mongo_kwargs=None, **kwargs): - import pymongo - - if client is None: - mongo_kwargs = mongo_kwargs if mongo_kwargs is not None else {} - self._client = pymongo.MongoClient(**mongo_kwargs) - else: - self._client = client - self._db = self._client.get_database(database) - self._collection = self._db.get_collection(collection) + def __init__(self, name=None, collection, **kwargs): + self._collection = collection self._name = name self._key = type(self).__name__ + '::name' super().__init__(**kwargs) @@ -83,19 +74,14 @@ class MongoDict(MongoCollection, SyncedAttrDict): ---------- name: str The name of the collection (Default value = None). - client: object - A Mongo client. - mongo_kwargs: dict - kwargs arguments passed through to the `pymongo.MongoClient` function. - database : string - Name of database (Default value = 'signac_db'). - collection : string - Name of collection (Default value = 'collection') + collection : object + A pymongo.Collection instance data: mapping, optional The intial data pass to MongoDict. Defaults to `dict()` parent: object, optional A parent instance of MongoDict or None (Default value = None). """ + pass @@ -127,14 +113,8 @@ class MongoList(MongoCollection, SyncedList): ---------- name: str The name of the collection (Default value = None). - client: - A Mongo client. - Mongo_kwargs: dict - kwargs arguments passed through to the `pymongo.MongoClient` function. - database : string - Name of database (Default value = 'signac_db'). - collection : string - Name of collection (Default value = 'collection') + collection : object + A pymongo.Collection instance data: mapping, optional The intial data pass to MongoList. Defaults to `list()` parent: object, optional diff --git a/signac/core/rediscollection.py b/signac/core/collection_redis.py similarity index 84% rename from signac/core/rediscollection.py rename to signac/core/collection_redis.py index 7cc387382..d9c161b49 100644 --- a/signac/core/rediscollection.py +++ b/signac/core/collection_redis.py @@ -18,14 +18,8 @@ class RedisCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, client=None, redis_kwargs=None, **kwargs): - import redis - - if client is None: - redis_kwargs = redis_kwargs if redis_kwargs is not None else {} - self._client = redis.Redis(**redis_kwargs) - else: - self._client = client + def __init__(self, name=None, client=None **kwargs): + self._client = client self._name = name super().__init__(**kwargs) if (name is None) == (self._parent is None): @@ -36,15 +30,11 @@ def __init__(self, name=None, client=None, redis_kwargs=None, **kwargs): def _load(self): """Load the data from a Redis-database.""" blob = self._client.get(self._name) - return json.loads(blob) if blob is not None else blob + return None if blob is None else json.loads(blob) def _sync(self): """Write the data from Redis-database.""" - data = self.to_base() - # Serialize data: - blob = json.dumps(data).encode() - - self._client.set(self._name, blob) + self._client.set(self._name, json.dumps(self.to_base()).encode()) class RedisDict(RedisCollection, SyncedAttrDict): @@ -85,8 +75,6 @@ class RedisDict(RedisCollection, SyncedAttrDict): The name of the collection (Default value = None). client: object A redis client. - redis_kwargs: dict - kwargs arguments passed through to the `redis.Redis` function. data: mapping, optional The intial data pass to RedisDict. Defaults to `dict()` parent: object, optional @@ -126,8 +114,6 @@ class RedisList(RedisCollection, SyncedList): The name of the collection (Default value = None). client: object A redis client. - redis_kwargs: dict - kwargs arguments passed through to the `redis.Redis` function. data: mapping, optional The intial data pass to RedisList. Defaults to `list()` parent: object, optional diff --git a/signac/core/zarrcollection.py b/signac/core/collection_zarr.py similarity index 91% rename from signac/core/zarrcollection.py rename to signac/core/collection_zarr.py index f835a45c2..ea712b22c 100644 --- a/signac/core/zarrcollection.py +++ b/signac/core/collection_zarr.py @@ -16,11 +16,10 @@ class ZarrCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, store=None, **kwargs): - import zarr + def __init__(self, name=None, group=None, **kwargs): import numcodecs # zarr depends on numcodecs - self._root = zarr.group(store=store) + self._root = group self._name = name self._object_codec = numcodecs.JSON() super().__init__(**kwargs) @@ -32,16 +31,13 @@ def __init__(self, name=None, store=None, **kwargs): def _load(self): """Load the data from a Radis-database.""" try: - dataset = self._root[self._name] - data = dataset[0] + return self._root[self._name][0] except KeyError: - data = None - return data + return None def _sync(self): """Write the data from Radis-database.""" data = self.to_base() - # Serialize data: dataset = self._root.require_dataset( self._name, overwrite=True, shape=1, dtype='object', object_codec=self._object_codec) dataset[0] = data @@ -85,8 +81,8 @@ class ZarrDict(ZarrCollection, SyncedAttrDict): The name of the collection (Default value = None). data: mapping, optional The intial data pass to ZarrDict. Defaults to `dict()`. - store: mapping - A zarr store to synchronise the data + group: object + A zarr.hierarchy.Group instance parent: object, optional A parent instance of ZarrDict or None (Default value = None). """ @@ -124,8 +120,8 @@ class ZarrList(ZarrCollection, SyncedList): The name of the collection. data: mapping, optional The intial data pass to ZarrList. Defaults to `list()`. - store: mapping - A zarr store to synchronise the data + group: object + A zarr.hierarchy.Group instance parent: object, optional A parent instance of ZarrList or None (Default value = None). """ From 3dc74131fa3a9b663abb0c3cbd68971d9641ebd4 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 12 Aug 2020 02:01:57 +0530 Subject: [PATCH 23/40] Documentation changes --- signac/core/collection_mongodb.py | 2 +- signac/core/collection_redis.py | 2 +- signac/core/collection_zarr.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 26ae88b3a..96c3fde5f 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -16,7 +16,7 @@ class MongoDBCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, collection, **kwargs): + def __init__(self, collection, name=None, **kwargs): self._collection = collection self._name = name self._key = type(self).__name__ + '::name' diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index d9c161b49..262496a50 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -18,7 +18,7 @@ class RedisCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, client=None **kwargs): + def __init__(self, name=None, client=None, **kwargs): self._client = client self._name = name super().__init__(**kwargs) diff --git a/signac/core/collection_zarr.py b/signac/core/collection_zarr.py index ea712b22c..9482d3f23 100644 --- a/signac/core/collection_zarr.py +++ b/signac/core/collection_zarr.py @@ -29,14 +29,14 @@ def __init__(self, name=None, group=None, **kwargs): "parent or name must be None, but not both.") def _load(self): - """Load the data from a Radis-database.""" + """Load the data.""" try: return self._root[self._name][0] except KeyError: return None def _sync(self): - """Write the data from Radis-database.""" + """Write the data.""" data = self.to_base() dataset = self._root.require_dataset( self._name, overwrite=True, shape=1, dtype='object', object_codec=self._object_codec) From 6ff7ecb664e9d9afca290193fdfacc6946c92494 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 12 Aug 2020 02:03:40 +0530 Subject: [PATCH 24/40] Change name MongoCollection to MongoDBCollection --- signac/core/collection_mongodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 96c3fde5f..756eaa512 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -38,7 +38,7 @@ def _sync(self): self._collection.replace_one({self._key: self._name}, data_to_insert, True) -class MongoDict(MongoCollection, SyncedAttrDict): +class MongoDict(MongoDBCollection, SyncedAttrDict): """A dict-like mapping interface to a persistent Mongo-database. The MongoDict inherits from :class:`~core.collection_api.MongoCollection` @@ -85,7 +85,7 @@ class MongoDict(MongoCollection, SyncedAttrDict): pass -class MongoList(MongoCollection, SyncedList): +class MongoList(MongoDBCollection, SyncedList): """A non-string sequence interface to a persistent Mongo file. The MongoDict inherits from :class:`~core.synced_collection.SyncedCollection` From db8a96e2fe0e0b71c73e640b5c4d61da8c5c6dc6 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 00:46:05 +0530 Subject: [PATCH 25/40] Moved error for name and parent to synced_collection --- signac/core/collection_json.py | 7 ++----- signac/core/collection_mongodb.py | 8 +++----- signac/core/collection_redis.py | 7 +------ signac/core/collection_zarr.py | 7 +------ signac/core/synced_collection.py | 7 ++++++- signac/core/syncedattrdict.py | 2 +- 6 files changed, 14 insertions(+), 24 deletions(-) diff --git a/signac/core/collection_json.py b/signac/core/collection_json.py index 0a7e02121..d51ff970d 100644 --- a/signac/core/collection_json.py +++ b/signac/core/collection_json.py @@ -23,13 +23,10 @@ class JSONCollection(SyncedCollection): backend = __name__ # type: ignore def __init__(self, filename=None, write_concern=False, **kwargs): - self._filename = os.path.realpath(filename) if filename is not None else None + self._filename = None if filename is None else os.path.realpath(filename) self._write_concern = write_concern + kwargs['name'] = filename super().__init__(**kwargs) - if (filename is None) == (self._parent is None): - raise ValueError( - "Illegal argument combination, one of the two arguments, " - "parent or filename must be None, but not both.") def _load(self): """Load the data from a JSON-file.""" diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 756eaa512..51c67ea5d 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -17,14 +17,12 @@ class MongoDBCollection(SyncedCollection): backend = __name__ # type: ignore def __init__(self, collection, name=None, **kwargs): + import bson # for InvalidDocument + self._collection = collection - self._name = name + self._errors = bson.errors self._key = type(self).__name__ + '::name' super().__init__(**kwargs) - if (name is None) == (self._parent is None): - raise ValueError( - "Illegal argument combination, one of the two arguments, " - "parent or name must be None, but not both.") def _load(self): """Load the data from a Mongo-database.""" diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index 262496a50..c0327dd45 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -18,14 +18,9 @@ class RedisCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, client=None, **kwargs): + def __init__(self, client=None, **kwargs): self._client = client - self._name = name super().__init__(**kwargs) - if (name is None) == (self._parent is None): - raise ValueError( - "Illegal argument combination, one of the two arguments, " - "parent or name must be None, but not both.") def _load(self): """Load the data from a Redis-database.""" diff --git a/signac/core/collection_zarr.py b/signac/core/collection_zarr.py index 9482d3f23..5b14932e6 100644 --- a/signac/core/collection_zarr.py +++ b/signac/core/collection_zarr.py @@ -16,17 +16,12 @@ class ZarrCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, name=None, group=None, **kwargs): + def __init__(self, group=None, **kwargs): import numcodecs # zarr depends on numcodecs self._root = group - self._name = name self._object_codec = numcodecs.JSON() super().__init__(**kwargs) - if (name is None) == (self._parent is None): - raise ValueError( - "Illegal argument combination, one of the two arguments, " - "parent or name must be None, but not both.") def _load(self): """Load the data.""" diff --git a/signac/core/synced_collection.py b/signac/core/synced_collection.py index f275238a7..52eb7c81a 100644 --- a/signac/core/synced_collection.py +++ b/signac/core/synced_collection.py @@ -31,10 +31,15 @@ class SyncedCollection(Collection): backend = None - def __init__(self, parent=None): + def __init__(self, name=None, parent=None): self._data = None self._parent = parent + self._name = name self._suspend_sync_ = 0 + if (name is None) == (parent is None): + raise ValueError( + "Illegal argument combination, one of the two arguments, " + "parent or name must be None, but not both.") @classmethod def register(cls, *args): diff --git a/signac/core/syncedattrdict.py b/signac/core/syncedattrdict.py index ffc592eb6..206650450 100644 --- a/signac/core/syncedattrdict.py +++ b/signac/core/syncedattrdict.py @@ -35,7 +35,7 @@ class SyncedAttrDict(SyncedCollection, MutableMapping): dictionary representation, and if necessary construct a new SyncedAttrDict. """ - _PROTECTED_KEYS = ('_data', '_suspend_sync_', '_load', '_sync', '_parent') + _PROTECTED_KEYS = ('_data', '_name', '_suspend_sync_', '_load', '_sync', '_parent') VALID_KEY_TYPES = (str, int, bool, type(None)) From 064eafc2f2732347e4a4d430a637765c449cb8db Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 00:47:29 +0530 Subject: [PATCH 26/40] Removed pass --- signac/core/collection_json.py | 4 ---- signac/core/collection_mongodb.py | 6 +----- signac/core/collection_redis.py | 5 ----- signac/core/collection_zarr.py | 4 ---- 4 files changed, 1 insertion(+), 18 deletions(-) diff --git a/signac/core/collection_json.py b/signac/core/collection_json.py index d51ff970d..8fd70bacd 100644 --- a/signac/core/collection_json.py +++ b/signac/core/collection_json.py @@ -102,8 +102,6 @@ class JSONDict(JSONCollection, SyncedAttrDict): first, before replacing the original file (Default value = None). """ - pass - class JSONList(JSONCollection, SyncedList): """A non-string sequence interface to a persistent JSON file. @@ -142,7 +140,5 @@ class JSONList(JSONCollection, SyncedList): first, before replacing the original file (Default value = None). """ - pass - SyncedCollection.register(JSONDict, JSONList) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 51c67ea5d..ea8754ab7 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -17,7 +17,7 @@ class MongoDBCollection(SyncedCollection): backend = __name__ # type: ignore def __init__(self, collection, name=None, **kwargs): - import bson # for InvalidDocument + import bson # for InvalidDocument self._collection = collection self._errors = bson.errors @@ -80,8 +80,6 @@ class MongoDict(MongoDBCollection, SyncedAttrDict): A parent instance of MongoDict or None (Default value = None). """ - pass - class MongoList(MongoDBCollection, SyncedList): """A non-string sequence interface to a persistent Mongo file. @@ -119,7 +117,5 @@ class MongoList(MongoDBCollection, SyncedList): A parent instance of MongoList or None (Default value = None). """ - pass - SyncedCollection.register(MongoDict, MongoList) diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index c0327dd45..8aec36f36 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -76,8 +76,6 @@ class RedisDict(RedisCollection, SyncedAttrDict): A parent instance of RedisDict or None (Default value = None). """ - pass - class RedisList(RedisCollection, SyncedList): """A non-string sequence interface to a persistent Redis file. @@ -113,10 +111,7 @@ class RedisList(RedisCollection, SyncedList): The intial data pass to RedisList. Defaults to `list()` parent: object, optional A parent instance of RedisList or None (Default value = None). - """ - pass - SyncedCollection.register(RedisDict, RedisList) diff --git a/signac/core/collection_zarr.py b/signac/core/collection_zarr.py index 5b14932e6..69baf1884 100644 --- a/signac/core/collection_zarr.py +++ b/signac/core/collection_zarr.py @@ -82,8 +82,6 @@ class ZarrDict(ZarrCollection, SyncedAttrDict): A parent instance of ZarrDict or None (Default value = None). """ - pass - class ZarrList(ZarrCollection, SyncedList): """A non-string sequence interface to a persistent Zarr file. @@ -121,7 +119,5 @@ class ZarrList(ZarrCollection, SyncedList): A parent instance of ZarrList or None (Default value = None). """ - pass - SyncedCollection.register(ZarrDict, ZarrList) From a61a5ebb2e330ed1a9edf531b6ac3c03340767c6 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 00:49:09 +0530 Subject: [PATCH 27/40] Implemented deepcopy for MongodbCollection and RedisCollection --- signac/core/collection_mongodb.py | 9 ++++++++- signac/core/collection_redis.py | 3 +++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index ea8754ab7..5fdf7841a 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -33,7 +33,14 @@ def _sync(self): """Write the data from Mongo-database.""" data = self.to_base() data_to_insert = {self._key: self._name, 'data': data} - self._collection.replace_one({self._key: self._name}, data_to_insert, True) + try: + self._collection.replace_one({self._key: self._name}, data_to_insert, True) + except self._errors.InvalidDocument as err: + raise TypeError(str(err)) + + def __deepcopy__(self, memo): + return type(self)(client=self._collection, name=self._name, data=self.to_base() + parent=deepcopy(self._parent, memo)) class MongoDict(MongoDBCollection, SyncedAttrDict): diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index 8aec36f36..2c442da7e 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -31,6 +31,9 @@ def _sync(self): """Write the data from Redis-database.""" self._client.set(self._name, json.dumps(self.to_base()).encode()) + def __deepcopy__(self, memo): + return type(self)(client=self._client, name=self._name, data=self.to_base() + parent=deepcopy(self._parent, memo)) class RedisDict(RedisCollection, SyncedAttrDict): """A dict-like mapping interface to a persistent Redis-database. From 25b1139bdcd9c7d18cfdebb021f20b48d372ef2b Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 00:50:55 +0530 Subject: [PATCH 28/40] Moved testdata to confest.py --- signac/core/collection_mongodb.py | 2 +- signac/core/collection_redis.py | 3 ++- tests/conftest.py | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 5fdf7841a..5f22f4ea4 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -39,7 +39,7 @@ def _sync(self): raise TypeError(str(err)) def __deepcopy__(self, memo): - return type(self)(client=self._collection, name=self._name, data=self.to_base() + return type(self)(client=self._collection, name=self._name, data=self.to_base(), parent=deepcopy(self._parent, memo)) diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index 2c442da7e..ccecfde81 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -32,9 +32,10 @@ def _sync(self): self._client.set(self._name, json.dumps(self.to_base()).encode()) def __deepcopy__(self, memo): - return type(self)(client=self._client, name=self._name, data=self.to_base() + return type(self)(client=self._client, name=self._name, data=self.to_base(), parent=deepcopy(self._parent, memo)) + class RedisDict(RedisCollection, SyncedAttrDict): """A dict-like mapping interface to a persistent Redis-database. diff --git a/tests/conftest.py b/tests/conftest.py index e94b7439f..b6a9210a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from packaging import version import signac import pytest +import uuid @contextmanager @@ -11,3 +12,8 @@ def deprecated_in_version(version_string): yield else: yield + + +@pytest.fixture +def testdata(): + return str(uuid.uuid4()) \ No newline at end of file From 3bea2601d3f9e3761583dc0085641605950922b0 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 00:54:01 +0530 Subject: [PATCH 29/40] Moved backend test to specific file --- signac/core/collection_mongodb.py | 2 + signac/core/collection_redis.py | 1 + tests/conftest.py | 2 +- tests/test_mongo_collection.py | 57 +++++++++ tests/test_redis_collection.py | 52 ++++++++ tests/test_synced_collection.py | 201 ++---------------------------- tests/test_zarr_collection.py | 51 ++++++++ 7 files changed, 172 insertions(+), 194 deletions(-) create mode 100644 tests/test_mongo_collection.py create mode 100644 tests/test_redis_collection.py create mode 100644 tests/test_zarr_collection.py diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 5f22f4ea4..77962a064 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -6,6 +6,8 @@ This implements the MongoDB-backend for SyncedCollection API by implementing sync and load methods. """ +from copy impport deepcopy + from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict from .synced_list import SyncedList diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index ccecfde81..6f87e7ba8 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -7,6 +7,7 @@ implementing sync and load methods. """ import json +from copy import deepcopy from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict diff --git a/tests/conftest.py b/tests/conftest.py index b6a9210a4..fe2f608e8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,4 +16,4 @@ def deprecated_in_version(version_string): @pytest.fixture def testdata(): - return str(uuid.uuid4()) \ No newline at end of file + return str(uuid.uuid4()) diff --git a/tests/test_mongo_collection.py b/tests/test_mongo_collection.py new file mode 100644 index 000000000..b4c093840 --- /dev/null +++ b/tests/test_mongo_collection.py @@ -0,0 +1,57 @@ +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +import pytest + +from signac.core.collection_mongodb import MongoDict +from signac.core.collection_mongodb import MongoList +from test_synced_collection import TestJSONDict +from test_synced_collection import TestJSONList + +try: + import pymongo + try: + # test the mongodb server + MongoClient = pymongo.MongoClient() + tmp_collection = MongoClient['test_db']['test'] + tmp_collection.insert_one({'test': '0'}) + ret = tmp_collection.find_one({'test': '0'}) + assert ret['test'] == '0' + tmp_collection.drop() + PYMONGO = True + except (pymongo.errors.ServerSelectionTimeoutError, AssertionError): + PYMONGO = False +except ImportError: + PYMONGO = False + + +@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') +class TestMongoDict(TestJSONDict): + + @pytest.fixture + def synced_dict(self, request): + self._client = MongoClient + self._name = 'test' + self._collection = self._client.test_db.test_dict + yield MongoDict(name=self._name, collection= self._collection) + self._collection.drop() + + def store(self, data): + data_to_insert = {'MongoDict::name': self._name, 'data': data} + self._collection.replace_one({'MongoDict::name': self._name}, data_to_insert) + + +@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') +class TestMongoList(TestJSONList): + + @pytest.fixture + def synced_list(self, request): + self._client = MongoClient + self._name = 'test' + self._collection = self._client.test_db.test_list + yield MongoList(name=self._name, collection=self._collection) + self._collection.drop() + + def store(self, data): + data_to_insert = {'MongoList::name': self._name, 'data': data} + self._collection.replace_one({'MongoList::name': self._name}, data_to_insert) diff --git a/tests/test_redis_collection.py b/tests/test_redis_collection.py new file mode 100644 index 000000000..4c70cb140 --- /dev/null +++ b/tests/test_redis_collection.py @@ -0,0 +1,52 @@ +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +import pytest + +from signac.core.collection_redis import RedisDict +from signac.core.collection_redis import RedisList +from test_synced_collection import TestJSONDict +from test_synced_collection import TestJSONList + +try: + import redis + try: + # try to connect to server + RedisClient = redis.Redis() + test_key = str(uuid.uuid4()) + RedisClient.set(test_key, 0) + assert RedisClient.get(test_key) == b'0' # redis store data as bytes + RedisClient.delete(test_key) + REDIS = True + except (redis.exceptions.ConnectionError, AssertionError): + REDIS = False +except ImportError: + REDIS = False + + +@pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') +class TestRedisDict(TestJSONDict): + + @pytest.fixture + def synced_dict(self, request): + self._client = RedisClient + request.addfinalizer(self._client.flushall) + self._name = 'test' + yield RedisDict(name=self._name, client=self._client) + + def store(self, data): + self._client.set(self._name, json.dumps(data).encode()) + + +@pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') +class TestRedisList(TestJSONList): + + @pytest.fixture + def synced_list(self, request): + self._client = RedisClient + request.addfinalizer(self._client.flushall) + self._name = 'test' + yield RedisList(name=self._name, client=self._client) + + def store(self, data): + self._client.set(self._name, json.dumps(data).encode()) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 8ae494b77..5cace5d55 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -2,23 +2,16 @@ # All rights reserved. # This software is licensed under the BSD 3-Clause License. import pytest -import uuid import os import json from tempfile import TemporaryDirectory from collections.abc import MutableMapping from collections.abc import MutableSequence -from copy import copy +from copy import deepcopy from signac.core.synced_list import SyncedCollection -from signac.core.jsoncollection import JSONDict -from signac.core.jsoncollection import JSONList -from signac.core.zarrcollection import ZarrDict -from signac.core.zarrcollection import ZarrList -from signac.core.rediscollection import RedisDict -from signac.core.rediscollection import RedisList -from signac.core.mongodbcollection import MongoDict -from signac.core.mongodbcollection import MongoList +from signac.core.collection_json import JSONDict +from signac.core.collection_json import JSONList from signac.errors import InvalidKeyError from signac.errors import KeyTypeError @@ -28,56 +21,12 @@ except ImportError: NUMPY = False -try: - import zarr - import numcodecs # zarr depends on numcodecs - ZARR = True -except ImportError: - ZARR = False - -try: - import redis - try: - # try to connect to server - RedisClient = redis.Redis() - test_key = str(uuid.uuid4()) - RedisClient.set(test_key, 0) - assert RedisClient.get(test_key) == b'0' # redis store data as bytes - RedisClient.delete(test_key) - REDIS = True - except (redis.exceptions.ConnectionError, AssertionError): - REDIS = False -except ImportError: - REDIS = False - -try: - import pymongo - import bson # required for invalid data error in test_write_invalid_type - try: - # test the mongodb server - MongoClient = pymongo.MongoClient() - tmp_collection = MongoClient['test_db']['test'] - tmp_collection.insert_one({'test': '0'}) - ret = tmp_collection.find_one({'test': '0'}) - assert ret['test'] == '0' - tmp_collection.drop() - PYMONGO = True - except (pymongo.errors.ServerSelectionTimeoutError, AssertionError): - PYMONGO = False -except ImportError: - PYMONGO = False - FN_JSON = 'test.json' -@pytest.fixture -def testdata(): - return str(uuid.uuid4()) - +class TestJSONCollectionBase: -class TestSyncedCollectionBase: - - # this fixture sets temprary directory for tests + # this fixture sets temporary directory for tests @pytest.fixture(autouse=True) def synced_collection(self): self._tmp_dir = TemporaryDirectory(prefix='synced_collection_') @@ -87,36 +36,11 @@ def synced_collection(self): def test_from_base_json(self): sd = SyncedCollection.from_base(filename=self._fn_, - data={'a': 0}, backend='signac.core.jsoncollection') + data={'a': 0}, backend='signac.core.collection_json') assert isinstance(sd, JSONDict) assert 'a' in sd assert sd['a'] == 0 - @pytest.mark.skipif(not ZARR, reason='test requires the zarr package') - def test_from_base_zarr(self): - sd = SyncedCollection.from_base(name='test', - data={'a': 0}, backend='signac.core.zarrcollection') - assert isinstance(sd, ZarrDict) - assert 'a' in sd - assert sd['a'] == 0 - - @pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package.') - def test_from_base_mongo(self): - sd = SyncedCollection.from_base(name='test', data={'a': 0}, client=MongoClient, - backend='signac.core.mongodbcollection') - assert isinstance(sd, MongoDict) - assert 'a' in sd - assert sd['a'] == 0 - - @pytest.mark.skipif(not REDIS, reason='test requires the redis package ' - 'and running redis-server') - def test_from_base_redis(self): - sd = SyncedCollection.from_base(name='test', data={'a': 0}, client=RedisClient, - backend='signac.core.rediscollection') - assert isinstance(sd, RedisDict) - assert 'a' in sd - assert sd['a'] == 0 - def test_from_base_no_backend(self): with pytest.raises(ValueError): SyncedCollection.from_base(data={'a': 0}) @@ -349,7 +273,7 @@ def test_call(self, synced_dict, testdata): def test_reopen(self, synced_dict, testdata): key = 'reopen' synced_dict[key] = testdata - synced_dict2 = copy(synced_dict) + synced_dict2 = deepcopy(synced_dict) synced_dict.sync() del synced_dict # possibly unsafe synced_dict2.load() @@ -597,7 +521,7 @@ def test_update_recursive(self, synced_list): synced_list.load() def test_reopen(self, synced_list, testdata): - synced_list2 = copy(synced_list) + synced_list2 = deepcopy(synced_list) synced_list.append(testdata) synced_list.sync() del synced_list # possibly unsafe @@ -672,112 +596,3 @@ def synced_list(self): self._backend_kwargs = {'filename': self._fn_, 'write_concern': True} yield JSONList(**self._backend_kwargs) self._tmp_dir.cleanup() - - -@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') -class TestZarrDict(TestJSONDict): - - @pytest.fixture(autouse=True) - def synced_dict(self): - self._tmp_dir = TemporaryDirectory(prefix='zarrdict_') - self._store = zarr.DirectoryStore(self._tmp_dir.name) - self._name = 'test' - yield ZarrDict(name=self._name, store=self._store) - self._tmp_dir.cleanup() - - def store(self, data): - dataset = zarr.group(self._store).require_dataset( - 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) - dataset[0] = data - - -@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') -class TestZarrList(TestJSONList): - - @pytest.fixture(autouse=True) - def synced_list(self): - self._tmp_dir = TemporaryDirectory(prefix='zarrlist_') - self._store = zarr.DirectoryStore(self._tmp_dir.name) - self._name = 'test' - yield ZarrList(name=self._name, store=self._store) - self._tmp_dir.cleanup() - - def store(self, data): - dataset = zarr.group(self._store).require_dataset( - 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) - dataset[0] = data - - -@pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') -class TestRedisDict(TestJSONDict): - - @pytest.fixture - def synced_dict(self, request): - self._client = RedisClient - request.addfinalizer(self._client.flushall) - self._name = 'test' - yield RedisDict(name=self._name, client=self._client) - - def store(self, data): - self._client.set(self._name, json.dumps(data).encode()) - - -@pytest.mark.skipif(not REDIS, reason='test requires the redis package and running redis-server') -class TestRedisList(TestJSONList): - - @pytest.fixture - def synced_list(self, request): - self._client = RedisClient - request.addfinalizer(self._client.flushall) - self._name = 'test' - yield RedisList(name=self._name, client=self._client) - - def store(self, data): - self._client.set(self._name, json.dumps(data).encode()) - - -@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') -class TestMongoDict(TestJSONDict): - - @pytest.fixture - def synced_dict(self, request): - self._client = MongoClient - self._name = 'test' - yield MongoDict(name=self._name, client=self._client, - database='test_db', collection='test_dict') - self._client.test_db.test_dict.drop() - - def store(self, data): - data_to_insert = {'MongoDict::name': self._name, 'data': data} - self._client.test_db.test_dict.replace_one({'MongoDict::name': self._name}, data_to_insert) - - def test_write_invalid_type(self, synced_dict, testdata): - # mongodict return InvalidDocument error for objects - class Foo(object): - pass - - key = 'write_invalid_type' - synced_dict[key] = testdata - assert len(synced_dict) == 1 - assert synced_dict[key] == testdata - d2 = Foo() - with pytest.raises(bson.errors.InvalidDocument): - synced_dict[key + '2'] = d2 - assert len(synced_dict) == 1 - assert synced_dict[key] == testdata - - -@pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') -class TestMongoList(TestJSONList): - - @pytest.fixture - def synced_list(self, request): - self._client = MongoClient - self._name = 'test' - yield MongoList(name=self._name, client=self._client, - database='test_db', collection='test_list') - self._client.test_db.test_list.drop() - - def store(self, data): - data_to_insert = {'MongoList::name': self._name, 'data': data} - self._client.test_db.test_list.replace_one({'MongoList::name': self._name}, data_to_insert) diff --git a/tests/test_zarr_collection.py b/tests/test_zarr_collection.py new file mode 100644 index 000000000..db109b03a --- /dev/null +++ b/tests/test_zarr_collection.py @@ -0,0 +1,51 @@ +# Copyright (c) 2020 The Regents of the University of Michigan +# All rights reserved. +# This software is licensed under the BSD 3-Clause License. +import pytest +from tempfile import TemporaryDirectory + +from signac.core.collection_zarr import ZarrDict +from signac.core.collection_zarr import ZarrList +from test_synced_collection import TestJSONDict +from test_synced_collection import TestJSONList + +try: + import zarr + import numcodecs # zarr depends on numcodecs + ZARR = True +except ImportError: + ZARR = False + + +@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') +class TestZarrDict(TestJSONDict): + + @pytest.fixture(autouse=True) + def synced_dict(self): + self._tmp_dir = TemporaryDirectory(prefix='zarrdict_') + self._group = zarr.group(zarr.DirectoryStore(self._tmp_dir.name)) + self._name = 'test' + yield ZarrDict(name=self._name, group=self._group) + self._tmp_dir.cleanup() + + def store(self, data): + dataset = self._group.require_dataset( + 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) + dataset[0] = data + + +@pytest.mark.skipif(not ZARR, reason='test requires the zarr package') +class TestZarrList(TestJSONList): + + @pytest.fixture(autouse=True) + def synced_list(self): + self._tmp_dir = TemporaryDirectory(prefix='zarrlist_') + self._group = zarr.group(zarr.DirectoryStore(self._tmp_dir.name)) + self._name = 'test' + yield ZarrList(name=self._name, group=self._group) + self._tmp_dir.cleanup() + + def store(self, data): + dataset = self._group.require_dataset( + 'test', overwrite=True, shape=1, dtype='object', object_codec=numcodecs.JSON()) + dataset[0] = data From 4789b37f92c58f1f1e03407b017b5389c2fa3916 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 00:56:28 +0530 Subject: [PATCH 30/40] Linting errors --- signac/core/collection_mongodb.py | 2 +- tests/test_mongo_collection.py | 2 +- tests/test_redis_collection.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 77962a064..9d35d81bf 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -6,7 +6,7 @@ This implements the MongoDB-backend for SyncedCollection API by implementing sync and load methods. """ -from copy impport deepcopy +from copy import deepcopy from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict diff --git a/tests/test_mongo_collection.py b/tests/test_mongo_collection.py index b4c093840..5767230a2 100644 --- a/tests/test_mongo_collection.py +++ b/tests/test_mongo_collection.py @@ -33,7 +33,7 @@ def synced_dict(self, request): self._client = MongoClient self._name = 'test' self._collection = self._client.test_db.test_dict - yield MongoDict(name=self._name, collection= self._collection) + yield MongoDict(name=self._name, collection=self._collection) self._collection.drop() def store(self, data): diff --git a/tests/test_redis_collection.py b/tests/test_redis_collection.py index 4c70cb140..cc1cca1d4 100644 --- a/tests/test_redis_collection.py +++ b/tests/test_redis_collection.py @@ -2,6 +2,8 @@ # All rights reserved. # This software is licensed under the BSD 3-Clause License. import pytest +import json +import uuid from signac.core.collection_redis import RedisDict from signac.core.collection_redis import RedisList From 813fe90d38ea370c2741e841c4e41a51ef09e60b Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 01:02:36 +0530 Subject: [PATCH 31/40] Error correction --- signac/core/collection_mongodb.py | 4 ++-- .../{test_mongo_collection.py => test_mongodb_collection.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/{test_mongo_collection.py => test_mongodb_collection.py} (100%) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 9d35d81bf..796a63e59 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -18,7 +18,7 @@ class MongoDBCollection(SyncedCollection): backend = __name__ # type: ignore - def __init__(self, collection, name=None, **kwargs): + def __init__(self, collection=None, **kwargs): import bson # for InvalidDocument self._collection = collection @@ -41,7 +41,7 @@ def _sync(self): raise TypeError(str(err)) def __deepcopy__(self, memo): - return type(self)(client=self._collection, name=self._name, data=self.to_base(), + return type(self)(collection=self._collection, name=self._name, data=self.to_base(), parent=deepcopy(self._parent, memo)) diff --git a/tests/test_mongo_collection.py b/tests/test_mongodb_collection.py similarity index 100% rename from tests/test_mongo_collection.py rename to tests/test_mongodb_collection.py From c9b659a16f22eaca36e24910cdbf6e82ac5790b2 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 01:25:30 +0530 Subject: [PATCH 32/40] Added __deepcopy__ for zarr --- signac/core/collection_zarr.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/signac/core/collection_zarr.py b/signac/core/collection_zarr.py index 69baf1884..8fc911704 100644 --- a/signac/core/collection_zarr.py +++ b/signac/core/collection_zarr.py @@ -6,6 +6,8 @@ This implements the Zarr-backend for SyncedCollection API by implementing sync and load methods. """ +from copy import deepcopy + from .synced_collection import SyncedCollection from .syncedattrdict import SyncedAttrDict from .synced_list import SyncedList @@ -37,6 +39,10 @@ def _sync(self): self._name, overwrite=True, shape=1, dtype='object', object_codec=self._object_codec) dataset[0] = data + def __deepcopy__(self, memo): + return type(self)(group=self._root, name=self._name, data=self.to_base(), + parent=deepcopy(self._parent, memo)) + class ZarrDict(ZarrCollection, SyncedAttrDict): """A dict-like mapping interface to a persistent Zarr-database. From a0f28aa2034736fc4a4ecef6555de3a4c4eccf6f Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 23:31:55 +0530 Subject: [PATCH 33/40] Changed name of MongoDict and MongoList to MongoDBDict and MongoDBList --- signac/core/collection_mongodb.py | 10 +++++----- tests/test_mongodb_collection.py | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 796a63e59..c7df89dcc 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -27,12 +27,12 @@ def __init__(self, collection=None, **kwargs): super().__init__(**kwargs) def _load(self): - """Load the data from a Mongo-database.""" + """Load the data from a MongoDB.""" blob = self._collection.find_one({self._key: self._name}) return blob['data'] if blob is not None else None def _sync(self): - """Write the data from Mongo-database.""" + """Write the data from MongoDB.""" data = self.to_base() data_to_insert = {self._key: self._name, 'data': data} try: @@ -45,7 +45,7 @@ def __deepcopy__(self, memo): parent=deepcopy(self._parent, memo)) -class MongoDict(MongoDBCollection, SyncedAttrDict): +class MongoDBDict(MongoDBCollection, SyncedAttrDict): """A dict-like mapping interface to a persistent Mongo-database. The MongoDict inherits from :class:`~core.collection_api.MongoCollection` @@ -90,7 +90,7 @@ class MongoDict(MongoDBCollection, SyncedAttrDict): """ -class MongoList(MongoDBCollection, SyncedList): +class MongoDBList(MongoDBCollection, SyncedList): """A non-string sequence interface to a persistent Mongo file. The MongoDict inherits from :class:`~core.synced_collection.SyncedCollection` @@ -127,4 +127,4 @@ class MongoList(MongoDBCollection, SyncedList): """ -SyncedCollection.register(MongoDict, MongoList) +SyncedCollection.register(MongoDBDict, MongoDBList) diff --git a/tests/test_mongodb_collection.py b/tests/test_mongodb_collection.py index 5767230a2..5511baab6 100644 --- a/tests/test_mongodb_collection.py +++ b/tests/test_mongodb_collection.py @@ -3,8 +3,8 @@ # This software is licensed under the BSD 3-Clause License. import pytest -from signac.core.collection_mongodb import MongoDict -from signac.core.collection_mongodb import MongoList +from signac.core.collection_mongodb import MongoDBDict +from signac.core.collection_mongodb import MongoDBList from test_synced_collection import TestJSONDict from test_synced_collection import TestJSONList @@ -26,14 +26,14 @@ @pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') -class TestMongoDict(TestJSONDict): +class TestMongoDBDict(TestJSONDict): @pytest.fixture def synced_dict(self, request): self._client = MongoClient self._name = 'test' self._collection = self._client.test_db.test_dict - yield MongoDict(name=self._name, collection=self._collection) + yield MongoDBDict(name=self._name, collection=self._collection) self._collection.drop() def store(self, data): @@ -42,14 +42,14 @@ def store(self, data): @pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') -class TestMongoList(TestJSONList): +class TestMongoDBList(TestJSONList): @pytest.fixture def synced_list(self, request): self._client = MongoClient self._name = 'test' self._collection = self._client.test_db.test_list - yield MongoList(name=self._name, collection=self._collection) + yield MongoDBList(name=self._name, collection=self._collection) self._collection.drop() def store(self, data): From c2e402e9510d573739f75c97e5598867986d59f1 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 13 Aug 2020 23:33:32 +0530 Subject: [PATCH 34/40] Changed documentation and __deepcopy__ in zarr collection --- signac/core/collection_zarr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/signac/core/collection_zarr.py b/signac/core/collection_zarr.py index 8fc911704..7250192e3 100644 --- a/signac/core/collection_zarr.py +++ b/signac/core/collection_zarr.py @@ -26,21 +26,21 @@ def __init__(self, group=None, **kwargs): super().__init__(**kwargs) def _load(self): - """Load the data.""" + """Load the data from zarr-store.""" try: return self._root[self._name][0] except KeyError: return None def _sync(self): - """Write the data.""" + """Write the data to zarr-store.""" data = self.to_base() dataset = self._root.require_dataset( self._name, overwrite=True, shape=1, dtype='object', object_codec=self._object_codec) dataset[0] = data def __deepcopy__(self, memo): - return type(self)(group=self._root, name=self._name, data=self.to_base(), + return type(self)(group=deepcopy(self._root, memo), name=self._name, data=self.to_base(), parent=deepcopy(self._parent, memo)) From d9b3304a80e727ee7b8ad4db20249efd8d677624 Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 15 Aug 2020 03:26:28 +0530 Subject: [PATCH 35/40] Added _psuedo_deepcopy for redis and mongodb --- signac/core/collection_mongodb.py | 2 +- signac/core/collection_redis.py | 2 +- tests/test_synced_collection.py | 12 ++++++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index c7df89dcc..e71318aff 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -40,7 +40,7 @@ def _sync(self): except self._errors.InvalidDocument as err: raise TypeError(str(err)) - def __deepcopy__(self, memo): + def _pseudo_deepcopy(self, memo=None): return type(self)(collection=self._collection, name=self._name, data=self.to_base(), parent=deepcopy(self._parent, memo)) diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index 6f87e7ba8..bd130b94a 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -32,7 +32,7 @@ def _sync(self): """Write the data from Redis-database.""" self._client.set(self._name, json.dumps(self.to_base()).encode()) - def __deepcopy__(self, memo): + def _pseudo_deepcopy(self, memo=None): return type(self)(client=self._client, name=self._name, data=self.to_base(), parent=deepcopy(self._parent, memo)) diff --git a/tests/test_synced_collection.py b/tests/test_synced_collection.py index 5cace5d55..0ba22c431 100644 --- a/tests/test_synced_collection.py +++ b/tests/test_synced_collection.py @@ -273,7 +273,11 @@ def test_call(self, synced_dict, testdata): def test_reopen(self, synced_dict, testdata): key = 'reopen' synced_dict[key] = testdata - synced_dict2 = deepcopy(synced_dict) + try: + synced_dict2 = deepcopy(synced_dict) + except TypeError: + # Use fallback implementation, deepcopy not supported by backend. + synced_dict2 = synced_dict._pseudo_deepcopy() synced_dict.sync() del synced_dict # possibly unsafe synced_dict2.load() @@ -521,7 +525,11 @@ def test_update_recursive(self, synced_list): synced_list.load() def test_reopen(self, synced_list, testdata): - synced_list2 = deepcopy(synced_list) + try: + synced_list2 = deepcopy(synced_list) + except TypeError: + # Use fallback implementation, deepcopy not supported by backend. + synced_list2 = synced_list._pseudo_deepcopy() synced_list.append(testdata) synced_list.sync() del synced_list # possibly unsafe From c8b82cf499f442af63867cfd92ac428f548cfd4a Mon Sep 17 00:00:00 2001 From: unknown Date: Sat, 15 Aug 2020 03:37:22 +0530 Subject: [PATCH 36/40] Changed name MongoX to MongoDBX in test_mongodb_collection --- tests/test_mongodb_collection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_mongodb_collection.py b/tests/test_mongodb_collection.py index 5511baab6..328ebc06f 100644 --- a/tests/test_mongodb_collection.py +++ b/tests/test_mongodb_collection.py @@ -37,8 +37,8 @@ def synced_dict(self, request): self._collection.drop() def store(self, data): - data_to_insert = {'MongoDict::name': self._name, 'data': data} - self._collection.replace_one({'MongoDict::name': self._name}, data_to_insert) + data_to_insert = {'MongoDBDict::name': self._name, 'data': data} + self._collection.replace_one({'MongoDBDict::name': self._name}, data_to_insert) @pytest.mark.skipif(not PYMONGO, reason='test requires the pymongo package and mongodb server') @@ -53,5 +53,5 @@ def synced_list(self, request): self._collection.drop() def store(self, data): - data_to_insert = {'MongoList::name': self._name, 'data': data} - self._collection.replace_one({'MongoList::name': self._name}, data_to_insert) + data_to_insert = {'MongoDBList::name': self._name, 'data': data} + self._collection.replace_one({'MongoDBList::name': self._name}, data_to_insert) From 62c346a0d32c1d5cf0938e38fd0427f3d1f2c64d Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 17 Aug 2020 03:08:40 +0530 Subject: [PATCH 37/40] Added docstring for _pseudo_deepcopy --- signac/core/collection_mongodb.py | 9 +++++++-- signac/core/collection_redis.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index e71318aff..d8a7a06ca 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -40,9 +40,14 @@ def _sync(self): except self._errors.InvalidDocument as err: raise TypeError(str(err)) - def _pseudo_deepcopy(self, memo=None): + def _pseudo_deepcopy(self): + """Return a copy of instance. + + It is a psuedo implementation for `deepcopy` because + `pymongo.Collection` do not support `deepcopy`. + """ return type(self)(collection=self._collection, name=self._name, data=self.to_base(), - parent=deepcopy(self._parent, memo)) + parent=deepcopy(self._parent)) class MongoDBDict(MongoDBCollection, SyncedAttrDict): diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index bd130b94a..26835235c 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -32,9 +32,14 @@ def _sync(self): """Write the data from Redis-database.""" self._client.set(self._name, json.dumps(self.to_base()).encode()) - def _pseudo_deepcopy(self, memo=None): + def _pseudo_deepcopy(self): + """Return a copy of instance. + + It is a psuedo implementation for `deepcopy` because + `redis.Redis` do not support `deepcopy`. + """ return type(self)(client=self._client, name=self._name, data=self.to_base(), - parent=deepcopy(self._parent, memo)) + parent=deepcopy(self._parent)) class RedisDict(RedisCollection, SyncedAttrDict): From 49f2dffa752c9dd7065bff617e2f71abfda46b65 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 17 Aug 2020 03:09:41 +0530 Subject: [PATCH 38/40] Removed trailing space --- signac/core/collection_mongodb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index d8a7a06ca..7dd2f5d71 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -43,8 +43,8 @@ def _sync(self): def _pseudo_deepcopy(self): """Return a copy of instance. - It is a psuedo implementation for `deepcopy` because - `pymongo.Collection` do not support `deepcopy`. + It is a psuedo implementation for `deepcopy` because + `pymongo.Collection` do not support `deepcopy`. """ return type(self)(collection=self._collection, name=self._name, data=self.to_base(), parent=deepcopy(self._parent)) From 663781ca03d8b475ed9a1a02f4252870c1d3a49d Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 17 Aug 2020 23:47:32 +0530 Subject: [PATCH 39/40] Changes the docstring of _pseudo_deepcopy --- signac/core/collection_mongodb.py | 2 +- signac/core/collection_redis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index 7dd2f5d71..a2b855d52 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -44,7 +44,7 @@ def _pseudo_deepcopy(self): """Return a copy of instance. It is a psuedo implementation for `deepcopy` because - `pymongo.Collection` do not support `deepcopy`. + `pymongo.Collection` does not support `deepcopy` method. """ return type(self)(collection=self._collection, name=self._name, data=self.to_base(), parent=deepcopy(self._parent)) diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index 26835235c..b340c1d5a 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -36,7 +36,7 @@ def _pseudo_deepcopy(self): """Return a copy of instance. It is a psuedo implementation for `deepcopy` because - `redis.Redis` do not support `deepcopy`. + `redis.Redis` does not support `deepcopy` method. """ return type(self)(client=self._client, name=self._name, data=self.to_base(), parent=deepcopy(self._parent)) From 36787edf64b8f486be460eaf131617e49e75d009 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 19 Aug 2020 01:56:32 +0530 Subject: [PATCH 40/40] Changes the order of Parameters in documentation and minor corrections --- signac/core/collection_json.py | 20 ++++++------- signac/core/collection_mongodb.py | 48 +++++++++++++++---------------- signac/core/collection_redis.py | 24 ++++++++-------- signac/core/collection_zarr.py | 21 +++++++------- 4 files changed, 57 insertions(+), 56 deletions(-) diff --git a/signac/core/collection_json.py b/signac/core/collection_json.py index 8fd70bacd..18bcd7de2 100644 --- a/signac/core/collection_json.py +++ b/signac/core/collection_json.py @@ -93,20 +93,20 @@ class JSONDict(JSONCollection, SyncedAttrDict): ---------- filename: str, optional The filename of the associated JSON file on disk (Default value = None). + write_concern: bool, optional + Ensure file consistency by writing changes back to a temporary file + first, before replacing the original file (Default value = None). data: mapping, optional The intial data pass to JSONDict. Defaults to `list()` parent: object, optional A parent instance of JSONDict or None (Default value = None). - write_concern: bool, optional - Ensure file consistency by writing changes back to a temporary file - first, before replacing the original file (Default value = None). """ class JSONList(JSONCollection, SyncedList): """A non-string sequence interface to a persistent JSON file. - The JSONDict inherits from :class:`~core.collection_api.SyncedCollection` + The JSONList inherits from :class:`~core.collection_api.SyncedCollection` and :class:`~core.syncedlist.SyncedList`. .. code-block:: python @@ -129,15 +129,15 @@ class JSONList(JSONCollection, SyncedList): Parameters ---------- - filename: str + filename: str, optional The filename of the associated JSON file on disk (Default value = None). - data: non-str Sequence - The intial data pass to JSONDict - parent: object - A parent instance of JSONDict or None (Default value = None). - write_concern: bool + write_concern: bool, optional Ensure file consistency by writing changes back to a temporary file first, before replacing the original file (Default value = None). + data: non-str Sequence, optional + The intial data pass to JSONList + parent: object, optional + A parent instance of JSONList or None (Default value = None). """ diff --git a/signac/core/collection_mongodb.py b/signac/core/collection_mongodb.py index a2b855d52..83897e71c 100644 --- a/signac/core/collection_mongodb.py +++ b/signac/core/collection_mongodb.py @@ -53,12 +53,12 @@ def _pseudo_deepcopy(self): class MongoDBDict(MongoDBCollection, SyncedAttrDict): """A dict-like mapping interface to a persistent Mongo-database. - The MongoDict inherits from :class:`~core.collection_api.MongoCollection` + The MongoDBDict inherits from :class:`~core.collection_api.MongoCollection` and :class:`~core.syncedattrdict.SyncedAttrDict`. .. code-block:: python - doc = MongoDict('data') + doc = MongoDBDict('data') doc['foo'] = "bar" assert doc.foo == doc['foo'] == "bar" assert 'foo' in doc @@ -74,36 +74,36 @@ class MongoDBDict(MongoDBCollection, SyncedAttrDict): .. warning:: - While the MongoDict object behaves like a dictionary, there are + While the MongoDBDict object behaves like a dictionary, there are important distinctions to remember. In particular, because operations are reflected as changes to an underlying database, copying (even deep - copying) a MongoDict instance may exhibit unexpected behavior. If a + copying) a MongoDBDict instance may exhibit unexpected behavior. If a true copy is required, you should use the `to_base()` method to get a - dictionary representation, and if necessary construct a new MongoDict - instance: `new_dict = MongoDict(old_dict.to_base())`. + dictionary representation, and if necessary construct a new MongoDBDict + instance: `new_dict = MongoDBDict(old_dict.to_base())`. Parameters ---------- - name: str - The name of the collection (Default value = None). - collection : object - A pymongo.Collection instance + collection : object, optional + A pymongo.Collection instance. data: mapping, optional - The intial data pass to MongoDict. Defaults to `dict()` + The intial data pass to MongoDBDict. Defaults to `dict()`. + name: str, optional + The name of the collection (Default value = None). parent: object, optional - A parent instance of MongoDict or None (Default value = None). + A parent instance of MongoDBDict (Default value = None). """ class MongoDBList(MongoDBCollection, SyncedList): """A non-string sequence interface to a persistent Mongo file. - The MongoDict inherits from :class:`~core.synced_collection.SyncedCollection` + The MongoDBList inherits from :class:`~core.synced_collection.SyncedCollection` and :class:`~core.syncedlist.SyncedList`. .. code-block:: python - synced_list = MongoList('data') + synced_list = MongoDBList('data') synced_list.append("bar") assert synced_list[0] == "bar" assert len(synced_list) == 1 @@ -111,24 +111,24 @@ class MongoDBList(MongoDBCollection, SyncedList): .. warning:: - While the MongoList object behaves like a list, there are + While the MongoDBList object behaves like a list, there are important distinctions to remember. In particular, because operations are reflected as changes to an underlying database, copying (even deep - copying) a MongoList instance may exhibit unexpected behavior. If a + copying) a MongoDBList instance may exhibit unexpected behavior. If a true copy is required, you should use the `to_base()` method to get a - dictionary representation, and if necessary construct a new MongoList - instance: `new_list = MongoList(old_list.to_base())`. + dictionary representation, and if necessary construct a new MongoDBList + instance: `new_list = MongoDBList(old_list.to_base())`. Parameters ---------- - name: str + collection : object, optional + A pymongo.Collection instance (Default value = None). + data: non-str Sequence, optional + The intial data pass to MongoDBList. Defaults to `list()`. + name: str, optional The name of the collection (Default value = None). - collection : object - A pymongo.Collection instance - data: mapping, optional - The intial data pass to MongoList. Defaults to `list()` parent: object, optional - A parent instance of MongoList or None (Default value = None). + A parent instance of MongoDBList (Default value = None). """ diff --git a/signac/core/collection_redis.py b/signac/core/collection_redis.py index b340c1d5a..1dc9e504d 100644 --- a/signac/core/collection_redis.py +++ b/signac/core/collection_redis.py @@ -76,21 +76,21 @@ class RedisDict(RedisCollection, SyncedAttrDict): Parameters ---------- - name: str - The name of the collection (Default value = None). - client: object - A redis client. + client: object, optional + A redis client (Default value = None). data: mapping, optional The intial data pass to RedisDict. Defaults to `dict()` + name: str, optional + The name of the collection (Default value = None). parent: object, optional - A parent instance of RedisDict or None (Default value = None). + A parent instance of RedisDict (Default value = None). """ class RedisList(RedisCollection, SyncedList): """A non-string sequence interface to a persistent Redis file. - The RedisDict inherits from :class:`~core.collection_api.SyncedCollection` + The RedisList inherits from :class:`~core.collection_api.SyncedCollection` and :class:`~core.syncedlist.SyncedList`. .. code-block:: python @@ -113,14 +113,14 @@ class RedisList(RedisCollection, SyncedList): Parameters ---------- - name: str - The name of the collection (Default value = None). - client: object - A redis client. - data: mapping, optional + client: object, optional + A redis client (Default value = None). + data: non-str Sequence, optional The intial data pass to RedisList. Defaults to `list()` + name: str, optional + The name of the collection (Default value = None). parent: object, optional - A parent instance of RedisList or None (Default value = None). + A parent instance of RedisList (Default value = None). """ diff --git a/signac/core/collection_zarr.py b/signac/core/collection_zarr.py index 7250192e3..3cfef408d 100644 --- a/signac/core/collection_zarr.py +++ b/signac/core/collection_zarr.py @@ -78,12 +78,12 @@ class ZarrDict(ZarrCollection, SyncedAttrDict): Parameters ---------- - name: str, optional - The name of the collection (Default value = None). + group: object, optional + A zarr.hierarchy.Group instance (Default value = None). data: mapping, optional The intial data pass to ZarrDict. Defaults to `dict()`. - group: object - A zarr.hierarchy.Group instance + name: str, optional + The name of the collection (Default value = None). parent: object, optional A parent instance of ZarrDict or None (Default value = None). """ @@ -92,7 +92,7 @@ class ZarrDict(ZarrCollection, SyncedAttrDict): class ZarrList(ZarrCollection, SyncedList): """A non-string sequence interface to a persistent Zarr file. - The ZarrDict inherits from :class:`~core.collection_api.ZarrCollection` + The ZarrList inherits from :class:`~core.collection_api.ZarrCollection` and :class:`~core.syncedlist.SyncedList`. .. code-block:: python @@ -115,12 +115,13 @@ class ZarrList(ZarrCollection, SyncedList): Parameters ---------- - name: str - The name of the collection. - data: mapping, optional + + group: object, optional + A zarr.hierarchy.Group instance (Default value = None). + data: non-str Sequence, optional The intial data pass to ZarrList. Defaults to `list()`. - group: object - A zarr.hierarchy.Group instance + name: str, optional + The name of the collection (Default value = None). parent: object, optional A parent instance of ZarrList or None (Default value = None). """