Merge pull request #2641 from cta-observatory/optional-deps

Make some dependencies optional

.github/workflows/ci.yml

@@ -39,36 +39,50 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: ubuntu-latest
+          - name: Linux (3.10, mamba)
+            os: ubuntu-latest
             python-version: "3.10"
             install-method: mamba
+            extras: tests,all
 
-          - os: ubuntu-latest
+          - name: Linux (3.10, pip)
+            os: ubuntu-latest
             python-version: "3.10"
             install-method: pip
+            extras: tests,all
 
-          - os: ubuntu-latest
+          - name: Linux (3.11, pip, minimal-dependencies)
+            os: ubuntu-latest
             python-version: "3.11"
-            install-method: mamba
+            install-method: pip
+            extras: tests
 
-          - os: ubuntu-latest
+          - name: Linux (3.12, mamba, coverage)
+            os: ubuntu-latest
             python-version: "3.12"
             install-method: mamba
             extra-args: ["codecov"]
+            extras: tests,all
 
-          - os: ubuntu-latest
+          - name: Linux (3.12, pip)
+            os: ubuntu-latest
             python-version: "3.12"
             install-method: pip
+            extras: tests,all
 
           # macos 13 image is x86-based
-          - os: macos-13
+          - name: macos (3.10, x86_64, pip)
+            os: macos-13
             python-version: "3.10"
             install-method: pip
+            extras: tests,all
 
           # macos 14 image is arm64 based
-          - os: macos-14
+          - name: macos (3.12, arm64, mamba)
+            os: macos-14
             python-version: "3.12"
             install-method: mamba
+            extras: tests,all
 
     defaults:
       run:
@@ -122,9 +136,10 @@ jobs:
         if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
+          EXTRAS: ${{ matrix.extras }}
         run: |
           python --version | grep "Python ${PYTHON_VERSION}"
-          pip install -e .[tests]
+          pip install -e ".[${EXTRAS}]"
           pip install ./test_plugin
           pip freeze
           pip list

docs/api-reference/visualization/bokeh.rst

@@ -0,0 +1,14 @@
+.. _visualization_bokeh:
+
+**********************************************************
+Bokeh-based visualization (`~ctapipe.visualization.bokeh`)
+**********************************************************
+
+.. currentmodule:: ctapipe.visualization.bokeh
+
+
+Reference/API
+=============
+
+.. automodapi:: ctapipe.visualization.bokeh
+    :no-inheritance-diagram:

docs/api-reference/visualization/index.rst

@@ -12,13 +12,19 @@ Introduction
 
 This module provides methods to display various information related to
 events and reconstruction, like Cherenkov Camera images, image
-parameterizations, etc. The default implementation uses MatPlotLib
-to render displays, but in the future more rendering methods may be supported.
+parameterizations, etc.
 
-The main classes are:
+For the classes defined at this level, ``matplotlib`` is required as optional
+dependency. There are also bokeh-based versions of these classes defined in :ref:`visualization_bokeh`.
 
-* `CameraDisplay`
-* `ArrayDisplay`
+
+Submodules
+==========
+
+.. toctree::
+  :maxdepth: 1
+
+  bokeh
 
 
 Reference/API

docs/changes/2641.api.rst

@@ -0,0 +1,16 @@
+The following dependencies are now optional:
+
+* eventio, used for ``ctapipe.io.SimTelEventSource``.
+* matplotlib, used ``ctapipe.visualiation.CameraDisplay``, ``ctapipe.visualiation.ArrayDisplay``,
+  and most default visualiation tasks, e.g. ``.peek()`` methods.
+* iminuit, used for the ``ctapipe.image.muon`` and ``ctapipe.reco.impact`` fitting routines.
+* bokeh, for ``ctapipe.visualiation.bokeh``
+
+Code that needs these dependencies will now raise ``ctapipe.exceptions.OptionalDependencyMissing``
+in case such functionality is used and the dependency in question is not installed.
+
+These packages will now longer be installed by default when using e.g. ``pip install ctapipe``.
+
+If you want to install ctapipe with all optional dependencies included, do ``pip install "ctapipe[all]"``.
+
+The ``conda`` package will also no longer directly depend on these packages.

docs/conftest.py

@@ -2,4 +2,6 @@
     "conf.py",
     "_build",
     "auto_examples",
+    "api-reference/instrument/camerageometry_example.py",
+    "api-reference/coordinates/plot_camera_frames.py",
 ]

docs/developer-guide/getting-started.rst

@@ -100,7 +100,7 @@ usable from anywhere, given you have activated the ``cta-dev`` conda environment
 
 .. code-block:: console
 
-    $ pip install -e .
+    $ pip install -e '.[dev]'
 
 Using the editable installation means you won't have to rerun the installation for
 simple code changes to take effect.

docs/user-guide/getting-started.rst

@@ -41,6 +41,22 @@ or with pip:
    $ pip install ctapipe
 
 
+``ctapipe`` has a number of optional dependencies that are not automatically installed
+when just installing ``ctapipe``.
+These are:
+
+- `matplotlib <https://matplotlib.org/>`_ for visualization, used in `~ctapipe.visualization` and several other places.
+- `bokeh <https://bokeh.org/>`_ for web-based, interactive visualizations, see `~ctapipe.visualization.bokeh`.
+- `iminuit <https://scikit-hep.org/iminuit/>`_ for fitting, used in `~ctapipe.reco.impact` and `~ctapipe.image.muon`.
+- `eventio <https://github.com/cta-observatory/pyeventio>`_ used for reading ``sim_telarray`` files in `~ctapipe.io.SimTelEventSource`.
+
+You can install them individually, or if you just want to get ``ctapipe`` with all optional dependencies, use:
+
+.. code-block:: console
+
+   $ pip install 'ctapipe[all]'
+
+
 How To Get a Specific Version
 -----------------------------
 

environment.yml

@@ -48,6 +48,5 @@ dependencies:
   - wheel
   - xz
   - zlib
-  - zstandard
   - eventio>=1.9.1
   - ffmpeg # for making movies in the documentation

pyproject.toml

@@ -31,14 +31,11 @@ requires-python = ">=3.10"
 
 dependencies = [
     "astropy >=5.3,<7.0.0a0",
-    "bokeh ~=3.0",
     "docutils",
-    "eventio >=1.9.1, <2.0.0a0",
-    "iminuit >=2",
     "joblib",
-    "matplotlib ~=3.0",
     "numba >=0.56",
     "numpy >=1.23,<3.0.0a0",
+    "packaging",
     "psutil",
     "pyyaml >=5.1",
     "requests",
@@ -47,12 +44,23 @@ dependencies = [
     "tables ~=3.4",
     "tqdm >=4.32",
     "traitlets ~=5.6",
-    "zstandard",
-    "packaging",
 ]
 
 [project.optional-dependencies]
+
+# all is with all optional *runtime* dependencies
+# use `dev` to get really all dependencies
+all = [
+    "bokeh ~=3.0",
+    "eventio >=1.9.1,<2.0.0a0",
+    "iminuit >=2",
+    "matplotlib ~=3.0",
+]
+
 tests = [
+    # at the moment, essentially all tests rely on test data from simtel
+    # it doesn't make sense to skip all of these.
+    "eventio >=1.9.1,<2.0.0a0",
     "h5py",
     "pandas",
     "pytest >= 7.0",
@@ -64,33 +72,31 @@ tests = [
 ]
 
 docs = [
-    "sphinx",
-    "pydata_sphinx_theme",
-    "sphinx_automodapi",
+    "ctapipe[all]",
+    "ffmpeg-python",
+    "graphviz",
+    "ipython",
+    "jupyter",
     "nbsphinx",
+    "notebook",
     "numpydoc",
+    "pandas",
+    "pydata_sphinx_theme",
+    "pypandoc",
+    "sphinx",
     "sphinx-design",
     "sphinx-gallery >= 0.16.0",
+    "sphinx_automodapi",
     "sphinxcontrib-bibtex",
-    "jupyter",
-    "notebook",
-    "graphviz",
-    "pandas",
-    "ipython",
-    "ffmpeg-python",
-    "pypandoc",
     "tomli; python_version < '3.11'",
 ]
 
 dev = [
+    "ctapipe[all,docs,tests]",
     "pre-commit",
     "setuptools_scm[toml]",
 ]
 
-all = [
-    # self-reference with all extras to simplify
-    "ctapipe[tests,docs,dev]",
-]
 
 [project.scripts]
 ctapipe-info = "ctapipe.tools.info:main"

src/ctapipe/atmosphere.py

@@ -19,6 +19,8 @@
 from astropy.table import QTable, Table
 from scipy.interpolate import interp1d
 
+from ctapipe.exceptions import OptionalDependencyMissing
+
 from .compat import COPY_IF_NEEDED
 
 __all__ = [
@@ -152,7 +154,10 @@ def peek(self):
         Draw quick plot of profile
         """
         # pylint: disable=import-outside-toplevel
-        import matplotlib.pyplot as plt
+        try:
+            import matplotlib.pyplot as plt
+        except ModuleNotFoundError:
+            raise OptionalDependencyMissing("matplotlib") from None
 
         fig, axis = plt.subplots(1, 3, constrained_layout=True, figsize=(10, 3))
 

src/ctapipe/conftest.py

@@ -486,6 +486,7 @@ def dl1_muon_output_file(dl1_tmp_path, dl1_muon_file):
     from ctapipe.tools.process import ProcessorTool
 
     output = dl1_tmp_path / "muon_output.dl1.h5"
+    pytest.importorskip("iminuit")
 
     # prevent running process multiple times in case of parallel tests
     with FileLock(output.with_suffix(output.suffix + ".lock")):

src/ctapipe/core/provenance.py

@@ -14,10 +14,12 @@
 import warnings
 from collections import UserList
 from contextlib import contextmanager
+from functools import cache
 from importlib import import_module
-from importlib.metadata import distributions, version
+from importlib.metadata import Distribution, distributions
 from os.path import abspath
 from pathlib import Path
+from types import ModuleType
 
 import psutil
 from astropy.time import Time
@@ -45,17 +47,59 @@
 ]
 
 
+@cache
+def modules_of_distribution(distribution: Distribution):
+    modules = distribution.read_text("top_level.txt")
+    if modules is None:
+        return None
+    return set(modules.splitlines())
+
+
+@cache
+def get_distribution_of_module(module: ModuleType | str):
+    """Get the package distribution for an imported module"""
+    if isinstance(module, str):
+        name = module
+        module = import_module(module)
+    else:
+        name = module.__name__
+
+    path = Path(module.__file__).absolute()
+
+    for dist in distributions():
+        modules = modules_of_distribution(dist)
+        if modules is None:
+            base = dist.locate_file("")
+            if dist.files is not None and any(path == base / f for f in dist.files):
+                return dist
+        elif name in modules:
+            return dist
+
+    raise ValueError(f"Could not find a distribution for module: {module}")
+
+
 def get_module_version(name):
+    """
+    Get the version of a python *module*, something you can import.
+
+    If the module does not expose a ``__version__`` attribute, this function
+    will try to determine the *distribution* of the module and return its
+    version.
+    """
+    # we try first with module.__version__
+    # to support editable installs
     try:
         module = import_module(name)
+    except ModuleNotFoundError:
+        return "not installed"
+
+    try:
         return module.__version__
     except AttributeError:
         try:
-            return version(name)
+            return get_distribution_of_module(module).version
         except Exception:
             return "unknown"
-    except ImportError:
-        return "not installed"
 
 
 class MissingReferenceMetadata(UserWarning):
@@ -351,15 +395,13 @@ def _sortkey(dist):
 
 
 def _get_system_provenance():
-    """return JSON string containing provenance for all things that are
+    """return a dict containing provenance for all things that are
     fixed during the runtime"""
 
     bits, linkage = platform.architecture()
 
     return dict(
         ctapipe_version=__version__,
-        ctapipe_resources_version=get_module_version("ctapipe_resources"),
-        eventio_version=get_module_version("eventio"),
         ctapipe_svc_path=os.getenv("CTAPIPE_SVC_PATH"),
         executable=sys.executable,
         platform=dict(

src/ctapipe/core/tests/test_provenance.py

@@ -76,3 +76,11 @@ def test_provenance_input_reference_meta(provenance: Provenance, dl1_file):
     assert "reference_meta" in input_meta
     assert "CTA PRODUCT ID" in input_meta["reference_meta"]
     Reference.from_dict(input_meta["reference_meta"])
+
+
+def test_get_distribution_of_module():
+    from ctapipe.core.provenance import get_distribution_of_module
+
+    assert get_distribution_of_module("yaml").name == "PyYAML"
+    assert get_distribution_of_module("sklearn").name == "scikit-learn"
+    assert get_distribution_of_module("ctapipe_test_plugin").version == "0.1.0"