Feature/synced collections

.circleci/ci-oldest-reqs.txt

@@ -6,6 +6,7 @@ h5py==2.8.0
 numpy==1.13.3
 packaging==15.0
 pandas==1.0.0
+pymongo==3.0.0
 pytest-cov==2.10.1
 pytest==6.2.1
 tables==3.3.0

.circleci/config.yml

@@ -46,6 +46,8 @@ jobs:
   linux-python-39: &linux-template
     docker:
       - image: circleci/python:3.9
+      - image: circleci/mongo:latest
+      - image: circleci/redis:latest
 
     environment:
       BENCHMARKS: "RUN"
@@ -138,16 +140,22 @@ jobs:
     <<: *linux-template
     docker:
       - image: circleci/python:3.8
+      - image: circleci/mongo:latest
+      - image: circleci/redis:latest
 
   linux-python-37:
     <<: *linux-template
     docker:
       - image: circleci/python:3.7
+      - image: circleci/mongo:latest
+      - image: circleci/redis:latest
 
   linux-python-36-oldest:
     <<: *linux-template
     docker:
       - image: circleci/python:3.6
+      - image: circleci/mongo:latest
+      - image: circleci/redis:latest
     environment:
       BENCHMARKS: "SKIP"
       DEPENDENCIES: "OLDEST"
@@ -180,7 +188,12 @@ jobs:
             ${PYTHON} -m pip install --progress-bar off -U pip>=20.3
             ${PYTHON} -m pip install --progress-bar off -U codecov
             ${PYTHON} -m pip install --progress-bar off -U -r requirements/requirements-test.txt
-            ${PYTHON} -m pip install --progress-bar off -U -r requirements/requirements-test-optional.txt
+
+            # For some reason Zarr doesn't install correctly on Windows (runs
+            # into pip SSL errors), so we skip that test.
+            grep -v zarr requirements-test-optional.txt > requirements-test-optional-windows.txt
+            ${PYTHON} -m pip install --progress-bar off -U -r requirements-test-optional-windows.txt
+
             ${PYTHON} -m pip install --progress-bar off -U -e .
       - run:
           name: Run tests

changelog.txt

@@ -13,6 +13,7 @@ next
 Added
 +++++
 
+ - New ``SyncedCollection`` class and subclasses to replace ``JSONDict`` with more general support for different types of resources (such as MongoDB collections or Redis databases) and more complete support for different data types synchronized with files (#196, #234, #249, #316, #383, #397, #465, #484). This change introduces a minor-backwards incompatible change; for users making direct use of signac buffering, the ``force_write`` parameter is no longer respected. If the argument is passed, a warning will now be raised to indicate that it is ignored and will be removed in signac 2.0.
  - Unified querying for state point and document filters using 'sp' and 'doc' as prefixes (#332, #514). This change introduces a minor backwards-incompatible change to the ``Collection`` index schema ('statepoint'->'sp'), but this does not affect any APIs, only indexes saved to file using a previous version of signac. Indexing APIs will be removed in signac 2.0.
 
 

doc/api.rst

@@ -206,3 +206,111 @@ signac.errors module
     :members:
     :undoc-members:
     :show-inheritance:
+
+synced\_collections package
+===========================
+
+Data Types
+----------
+
+synced\_collections.synced\_collection module
++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.data_types.synced_collection
+   :members:
+   :private-members:
+   :show-inheritance:
+
+synced\_collections.synced\_dict module
++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.data_types.synced_dict
+   :members:
+   :show-inheritance:
+
+synced\_collections.synced\_list module
++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.data_types.synced_list
+   :members:
+   :show-inheritance:
+
+Backends
+--------
+
+synced\_collections.backends.collection\_json module
++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.backends.collection_json
+   :members:
+   :show-inheritance:
+
+synced\_collections.backends.collection\_mongodb module
+++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.backends.collection_mongodb
+   :members:
+   :show-inheritance:
+
+synced\_collections.backends.collection\_redis module
+++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.backends.collection_redis
+   :members:
+   :show-inheritance:
+
+synced\_collections.backends.collection\_zarr module
++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.backends.collection_zarr
+   :members:
+   :show-inheritance:
+
+Buffers
+-------
+
+synced\_collections.buffers.buffered\_collection module
++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.buffers.buffered_collection
+   :members:
+   :private-members:
+   :show-inheritance:
+
+synced\_collections.buffers.file\_buffered\_collection module
++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.buffers.file_buffered_collection
+   :members:
+   :show-inheritance:
+
+synced\_collections.buffers.serialized\_file\_buffered\_collection module
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.buffers.serialized_file_buffered_collection
+   :members:
+   :show-inheritance:
+
+synced\_collections.buffers.memory\_buffered\_collection module
++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.buffers.memory_buffered_collection
+   :members:
+   :show-inheritance:
+
+Miscellaneous Modules
+---------------------
+
+synced\_collections.utils module
+++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.utils
+   :members:
+   :show-inheritance:
+
+synced\_collections.validators module
++++++++++++++++++++++++++++++++++++++
+
+.. automodule:: signac.synced_collections.validators
+   :members:
+   :show-inheritance:

doc/conf.py

@@ -316,6 +316,9 @@ def __getattr__(cls, name):
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "pymongo": ("https://pymongo.readthedocs.io/en/stable/", None),
-    "pandas": ("https://pandas.pydata.org/docs/", None),
-    "h5py": ("https://docs.h5py.org/en/stable/", None),
+    "pandas": ("https://pandas.pydata.org/pandas-docs/stable/", None),
+    "h5py": ("http://docs.h5py.org/en/stable/", None),
+    "zarr": ("https://zarr.readthedocs.io/en/stable", None),
+    "redis": ("https://redis-py.readthedocs.io/en/stable/", None),
+    "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None),
 }

requirements/requirements-test-optional.txt

@@ -2,4 +2,7 @@ h5py==3.1.0; implementation_name=='cpython'
 numpy==1.20.0
 pandas==1.2.1; implementation_name=='cpython'
 pymongo==3.11.2; implementation_name=='cpython'
+redis==3.5.3
+ruamel.yaml==0.16.12
 tables==3.6.1; implementation_name=='cpython'
+zarr==2.4.0; platform_system!='Windows'

setup.cfg

@@ -20,7 +20,7 @@ ignore = E123,E126,E203,E226,E241,E704,W503,W504
 match = ^((?!\.sync-zenodo-metadata|setup|benchmark|mpipool|connection|crypt|host|filesystems|indexing).)*\.py$
 match-dir = ^((?!\.|tests|configobj|db).)*$
 ignore-decorators = "deprecated"
-ignore = D105, D107, D203, D204, D213
+add-ignore = D105, D107, D203, D204, D213
 
 [mypy]
 ignore_missing_imports = True
@@ -34,8 +34,8 @@ omit =
 	*/signac/common/configobj/*.py
 
 [tool:pytest]
-filterwarnings = 
-	ignore: .*[The indexing module | get_statepoint] is deprecated.*: DeprecationWarning
+filterwarnings =
+	ignore: .*[The indexing module | get_statepoint | Use of.+as key] is deprecated.*: DeprecationWarning
 
 [bumpversion:file:setup.py]
 

signac/__init__.py

@@ -27,15 +27,21 @@
 from .contrib import filesystems as fs
 from .contrib import get_job, get_project, index, index_files, init_project
 from .core.h5store import H5Store, H5StoreManager
-from .core.jsondict import JSONDict
-from .core.jsondict import buffer_reads_writes as buffered
 from .core.jsondict import flush_all as flush
-from .core.jsondict import get_buffer_load, get_buffer_size
-from .core.jsondict import in_buffered_mode as is_buffered
 from .db import get_database
 from .diff import diff_jobs
+from .synced_collections.backends.collection_json import (
+    BufferedJSONAttrDict as JSONDict,
+)
 from .version import __version__
 
+# Alias some properties related to buffering into the signac namespace.
+buffered = JSONDict.buffer_backend
+is_buffered = JSONDict.backend_is_buffered
+get_buffer_load = JSONDict.get_current_buffer_size
+get_buffer_size = JSONDict.get_buffer_capacity
+set_buffer_size = JSONDict.set_buffer_capacity
+
 __all__ = [
     "__version__",
     "contrib",
@@ -69,6 +75,7 @@
     "flush",
     "get_buffer_size",
     "get_buffer_load",
+    "set_buffer_size",
     "JSONDict",
     "H5Store",
     "H5StoreManager",

signac/contrib/collection.py

@@ -16,6 +16,7 @@
 
 import argparse
 import io
+import json
 import logging
 import operator
 import re
@@ -24,7 +25,6 @@
 from math import isclose
 from numbers import Number
 
-from ..core import json
 from .filterparse import parse_filter_arg
 from .utility import _nested_dicts_to_dotted_keys, _to_hashable
 

signac/contrib/errors.py

@@ -28,8 +28,8 @@ class DestinationExistsError(Error, RuntimeError):
 
     Parameters
     ----------
-    destination :
-        The destination object causing the error.
+    destination : str
+        The destination causing the error.
 
     """
 

signac/contrib/filterparse.py

@@ -3,11 +3,10 @@
 # This software is licensed under the BSD 3-Clause License.
 """Parse the filter arguments."""
 
+import json
 import sys
 from collections.abc import Mapping
 
-from ..core import json
-
 
 def _print_err(msg=None):
     """Print the provided message to stderr.

signac/contrib/hashing.py

@@ -6,6 +6,8 @@
 import hashlib
 import json
 
+from ..synced_collections.utils import SyncedCollectionJSONEncoder
+
 # We must use the standard library json for exact consistency in formatting
 
 
@@ -27,7 +29,7 @@ def calc_id(spec):
         Encoded hash in hexadecimal format.
 
     """
-    blob = json.dumps(spec, sort_keys=True)
+    blob = json.dumps(spec, cls=SyncedCollectionJSONEncoder, sort_keys=True)
     m = hashlib.md5()
     m.update(blob.encode())
     return m.hexdigest()

signac/contrib/import_export.py

@@ -4,6 +4,7 @@
 """Provides features for importing and exporting data."""
 
 import errno
+import json
 import logging
 import os
 import re
@@ -16,7 +17,6 @@
 from tempfile import TemporaryDirectory
 from zipfile import ZIP_DEFLATED, ZipFile
 
-from ..core import json
 from .errors import DestinationExistsError, StatepointParsingError
 from .utility import _dotted_dict_to_nested_dicts, _mkdir_p
 
@@ -766,7 +766,7 @@ def _copy_to_job_workspace(src, job, copytree):
             raise DestinationExistsError(job)
         raise
     else:
-        job._init()
+        job.init()
     return dst