Added backends to synced_collection

requirements-dev.txt

@@ -11,3 +11,5 @@ pytest-subtests==0.3.1
 pydocstyle==5.0.2
 pytest-cov==2.10.0
 pymongo==3.10.1
+zarr==2.4.0
+redis==3.5.3

signac/core/jsoncollection.py → signac/core/collection_json.py

@@ -23,13 +23,10 @@ class JSONCollection(SyncedCollection):
     backend = __name__  # type: ignore
 
     def __init__(self, filename=None, write_concern=False, **kwargs):
-        self._filename = os.path.realpath(filename) if filename is not None else None
+        self._filename = None if filename is None else os.path.realpath(filename)
         self._write_concern = write_concern
+        kwargs['name'] = filename
         super().__init__(**kwargs)
-        if (filename is None) == (self._parent is None):
-            raise ValueError(
-                "Illegal argument combination, one of the two arguments, "
-                "parent or filename must be None, but not both.")
 
     def _load(self):
         """Load the data from a JSON-file."""
@@ -96,22 +93,20 @@ class JSONDict(JSONCollection, SyncedAttrDict):
     ----------
     filename: str, optional
         The filename of the associated JSON file on disk (Default value = None).
+    write_concern: bool, optional
+        Ensure file consistency by writing changes back to a temporary file
+        first, before replacing the original file (Default value = None).
     data: mapping, optional
         The intial data pass to JSONDict. Defaults to `list()`
     parent: object, optional
         A parent instance of JSONDict or None (Default value = None).
-    write_concern: bool, optional
-        Ensure file consistency by writing changes back to a temporary file
-        first, before replacing the original file (Default value = None).
     """
 
-    pass
-
 
 class JSONList(JSONCollection, SyncedList):
     """A non-string sequence interface to a persistent JSON file.
 
-    The JSONDict inherits from :class:`~core.collection_api.SyncedCollection`
+    The JSONList inherits from :class:`~core.collection_api.SyncedCollection`
     and :class:`~core.syncedlist.SyncedList`.
 
     .. code-block:: python
@@ -134,18 +129,16 @@ class JSONList(JSONCollection, SyncedList):
 
     Parameters
     ----------
-    filename: str
+    filename: str, optional
         The filename of the associated JSON file on disk (Default value = None).
-    data: non-str Sequence
-        The intial data pass to JSONDict
-    parent: object
-        A parent instance of JSONDict or None (Default value = None).
-    write_concern: bool
+    write_concern: bool, optional
         Ensure file consistency by writing changes back to a temporary file
         first, before replacing the original file (Default value = None).
+    data: non-str Sequence, optional
+        The intial data pass to JSONList
+    parent: object, optional
+        A parent instance of JSONList or None (Default value = None).
     """
 
-    pass
-
 
 SyncedCollection.register(JSONDict, JSONList)

signac/core/collection_mongodb.py

@@ -0,0 +1,135 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""Implements MongoDB-backend.
+
+This implements the MongoDB-backend for SyncedCollection API by
+implementing sync and load methods.
+"""
+from copy import deepcopy
+
+from .synced_collection import SyncedCollection
+from .syncedattrdict import SyncedAttrDict
+from .synced_list import SyncedList
+
+
+class MongoDBCollection(SyncedCollection):
+    """Implement sync and load using a MongoDB backend."""
+
+    backend = __name__  # type: ignore
+
+    def __init__(self, collection=None, **kwargs):
+        import bson  # for InvalidDocument
+
+        self._collection = collection
+        self._errors = bson.errors
+        self._key = type(self).__name__ + '::name'
+        super().__init__(**kwargs)
+
+    def _load(self):
+        """Load the data from a MongoDB."""
+        blob = self._collection.find_one({self._key: self._name})
+        return blob['data'] if blob is not None else None
+
+    def _sync(self):
+        """Write the data from MongoDB."""
+        data = self.to_base()
+        data_to_insert = {self._key: self._name, 'data': data}
+        try:
+            self._collection.replace_one({self._key: self._name}, data_to_insert, True)
+        except self._errors.InvalidDocument as err:
+            raise TypeError(str(err))
+
+    def _pseudo_deepcopy(self):
+        """Return a copy of instance.
+
+        It is a psuedo implementation for `deepcopy` because
+        `pymongo.Collection` does not support `deepcopy` method.
+        """
+        return type(self)(collection=self._collection, name=self._name, data=self.to_base(),
+                          parent=deepcopy(self._parent))
+
+
+class MongoDBDict(MongoDBCollection, SyncedAttrDict):
+    """A dict-like mapping interface to a persistent Mongo-database.
+
+    The MongoDBDict inherits from :class:`~core.collection_api.MongoCollection`
+    and :class:`~core.syncedattrdict.SyncedAttrDict`.
+
+    .. code-block:: python
+
+        doc = MongoDBDict('data')
+        doc['foo'] = "bar"
+        assert doc.foo == doc['foo'] == "bar"
+        assert 'foo' in doc
+        del doc['foo']
+
+    .. code-block:: python
+
+        >>> doc['foo'] = dict(bar=True)
+        >>> doc
+        {'foo': {'bar': True}}
+        >>> doc.foo.bar = False
+        {'foo': {'bar': False}}
+
+    .. warning::
+
+        While the MongoDBDict object behaves like a dictionary, there are
+        important distinctions to remember. In particular, because operations
+        are reflected as changes to an underlying database, copying (even deep
+        copying) a MongoDBDict instance may exhibit unexpected behavior. If a
+        true copy is required, you should use the `to_base()` method to get a
+        dictionary representation, and if necessary construct a new MongoDBDict
+        instance: `new_dict = MongoDBDict(old_dict.to_base())`.
+
+    Parameters
+    ----------
+    collection : object, optional
+        A pymongo.Collection instance.
+    data: mapping, optional
+        The intial data pass to MongoDBDict. Defaults to `dict()`.
+    name: str, optional
+        The name of the  collection (Default value = None).
+    parent: object, optional
+        A parent instance of MongoDBDict (Default value = None).
+    """
+
+
+class MongoDBList(MongoDBCollection, SyncedList):
+    """A non-string sequence interface to a persistent Mongo file.
+
+    The MongoDBList inherits from :class:`~core.synced_collection.SyncedCollection`
+    and :class:`~core.syncedlist.SyncedList`.
+
+    .. code-block:: python
+
+        synced_list = MongoDBList('data')
+        synced_list.append("bar")
+        assert synced_list[0] == "bar"
+        assert len(synced_list) == 1
+        del synced_list[0]
+
+    .. warning::
+
+        While the MongoDBList object behaves like a list, there are
+        important distinctions to remember. In particular, because operations
+        are reflected as changes to an underlying database, copying (even deep
+        copying) a MongoDBList instance may exhibit unexpected behavior. If a
+        true copy is required, you should use the `to_base()` method to get a
+        dictionary representation, and if necessary construct a new MongoDBList
+        instance: `new_list = MongoDBList(old_list.to_base())`.
+
+    Parameters
+    ----------
+    collection : object, optional
+        A pymongo.Collection instance (Default value = None).
+    data: non-str Sequence, optional
+        The intial data pass to MongoDBList. Defaults to `list()`.
+    name: str, optional
+        The name of the  collection (Default value = None).
+    parent: object, optional
+        A parent instance of MongoDBList (Default value = None).
+    """
+
+
+SyncedCollection.register(MongoDBDict, MongoDBList)

signac/core/collection_redis.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2020 The Regents of the University of Michigan
+# All rights reserved.
+# This software is licensed under the BSD 3-Clause License.
+"""Implements Redis-backend.
+
+This implements the Redis-backend for SyncedCollection API by
+implementing sync and load methods.
+"""
+import json
+from copy import deepcopy
+
+from .synced_collection import SyncedCollection
+from .syncedattrdict import SyncedAttrDict
+from .synced_list import SyncedList
+
+
+class RedisCollection(SyncedCollection):
+    """Implement sync and load using a Redis backend."""
+
+    backend = __name__  # type: ignore
+
+    def __init__(self, client=None, **kwargs):
+        self._client = client
+        super().__init__(**kwargs)
+
+    def _load(self):
+        """Load the data from a Redis-database."""
+        blob = self._client.get(self._name)
+        return None if blob is None else json.loads(blob)
+
+    def _sync(self):
+        """Write the data from Redis-database."""
+        self._client.set(self._name, json.dumps(self.to_base()).encode())
+
+    def _pseudo_deepcopy(self):
+        """Return a copy of instance.
+
+        It is a psuedo implementation for `deepcopy` because
+        `redis.Redis` does not support `deepcopy` method.
+        """
+        return type(self)(client=self._client, name=self._name, data=self.to_base(),
+                          parent=deepcopy(self._parent))
+
+
+class RedisDict(RedisCollection, SyncedAttrDict):
+    """A dict-like mapping interface to a persistent Redis-database.
+
+    The RedisDict inherits from :class:`~core.rediscollection.RedisCollection`
+    and :class:`~core.syncedattrdict.SyncedAttrDict`.
+
+    .. code-block:: python
+
+        doc = RedisDict('data')
+        doc['foo'] = "bar"
+        assert doc.foo == doc['foo'] == "bar"
+        assert 'foo' in doc
+        del doc['foo']
+
+    .. code-block:: python
+
+        >>> doc['foo'] = dict(bar=True)
+        >>> doc
+        {'foo': {'bar': True}}
+        >>> doc.foo.bar = False
+        {'foo': {'bar': False}}
+
+    .. warning::
+
+        While the RedisDict object behaves like a dictionary, there are
+        important distinctions to remember. In particular, because operations
+        are reflected as changes to an underlying database, copying (even deep
+        copying) a RedisDict instance may exhibit unexpected behavior. If a
+        true copy is required, you should use the `to_base()` method to get a
+        dictionary representation, and if necessary construct a new RedisDict
+        instance: `new_dict = RedisDict(old_dict.to_base())`.
+
+    Parameters
+    ----------
+    client: object, optional
+        A redis client (Default value = None).
+    data: mapping, optional
+        The intial data pass to RedisDict. Defaults to `dict()`
+    name: str, optional
+        The name of the  collection (Default value = None).
+    parent: object, optional
+        A parent instance of RedisDict (Default value = None).
+    """
+
+
+class RedisList(RedisCollection, SyncedList):
+    """A non-string sequence interface to a persistent Redis file.
+
+    The RedisList inherits from :class:`~core.collection_api.SyncedCollection`
+    and :class:`~core.syncedlist.SyncedList`.
+
+    .. code-block:: python
+
+        synced_list = RedisList('data')
+        synced_list.append("bar")
+        assert synced_list[0] == "bar"
+        assert len(synced_list) == 1
+        del synced_list[0]
+
+    .. warning::
+
+        While the RedisList object behaves like a list, there are
+        important distinctions to remember. In particular, because operations
+        are reflected as changes to an underlying database, copying (even deep
+        copying) a RedisList instance may exhibit unexpected behavior. If a
+        true copy is required, you should use the `to_base()` method to get a
+        dictionary representation, and if necessary construct a new RedisList
+        instance: `new_list = RedisList(old_list.to_base())`.
+
+    Parameters
+    ----------
+    client: object, optional
+        A redis client (Default value = None).
+    data: non-str Sequence, optional
+        The intial data pass to RedisList. Defaults to `list()`
+    name: str, optional
+        The name of the  collection (Default value = None).
+    parent: object, optional
+        A parent instance of RedisList (Default value = None).
+    """
+
+
+SyncedCollection.register(RedisDict, RedisList)