Refactoring for Hvplot batch plotting

requirements.txt

@@ -1,7 +1,13 @@
 pyyaml>=6.0
-pycodestyle>=2.8.0
-netCDF4>=1.5.3
+pycodestyle>=2.9.1
+netCDF4>=1.6.1
 matplotlib>=3.5.2
-cartopy>=0.20.2
-scikit-learn>=1.0.2
-xarray>=0.11.3
+cartopy>=0.21.1
+scikit-learn>=1.1.2
+xarray>=2022.6.0
+seaborn>=0.12.2
+hvplot>=0.8.2
+nbconvert>=6.5.4
+bokeh>=3.1.1
+geopandas>=0.13.2
+geoviews>=1.10.0

requirements_emc.txt

@@ -0,0 +1,7 @@
+pyyaml>=6.0
+pycodestyle>=2.8.0
+netCDF4>=1.5.3
+matplotlib>=3.5.2
+cartopy>=0.20.2
+scikit-learn>=1.0.2
+xarray>=0.11.3

src/eva/eva_driver.py

@@ -14,7 +14,7 @@
 from eva.utilities.timing import Timing
 from eva.data.data_driver import data_driver
 from eva.transforms.transform_driver import transform_driver
-from eva.plotting.emcpy.plot_tools.figure_driver import figure_driver
+from eva.plotting.batch.base.plot_tools.figure_driver import figure_driver
 from eva.data.data_collections import DataCollections
 from eva.utilities.utils import load_yaml_file
 import argparse

src/eva/eva_interactive.py

@@ -23,7 +23,7 @@
 from eva.transforms.arithmetic import arithmetic, generate_arithmetic_config
 from eva.transforms.accept_where import accept_where, generate_accept_where_config
 
-import eva.plotting.hvplot.interactive_plot_tools as plot
+import eva.plotting.interactive.interactive_plot_tools as plot
 
 
 # --------------------------------------------------------------------------------------------------

src/eva/plotting/batch/__init__.py

@@ -0,0 +1,9 @@
+# (C) Copyright 2023 United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+import os
+
+repo_directory = os.path.dirname(__file__)

src/eva/plotting/batch/base/__init__.py

@@ -0,0 +1,9 @@
+# (C) Copyright 2023 United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+import os
+
+repo_directory = os.path.dirname(__file__)

src/eva/plotting/batch/base/diagnostics/scatter.py

@@ -0,0 +1,103 @@
+from eva.eva_path import return_eva_path
+from eva.utilities.config import get
+from eva.utilities.utils import get_schema, update_object, slice_var_from_str
+import os
+import numpy as np
+
+from abc import ABC, abstractmethod
+
+# --------------------------------------------------------------------------------------------------
+
+
+class Scatter(ABC):
+
+    """Base class for creating scatter plots."""
+
+    def __init__(self, config, logger, dataobj):
+
+        """
+        Creates a scatter plot on a map based on the provided configuration.
+
+        Args:
+            config (dict): A dictionary containing the configuration for the scatter plot on a map.
+            logger (Logger): An instance of the logger for logging messages.
+            dataobj: An instance of the data object containing input data.
+
+        This class initializes and configures a scatter plot on a map based on the provided
+        configuration. The scatter plot is created using a declarative plotting library from EMCPy
+        (https://github.com/NOAA-EMC/emcpy).
+
+        Example:
+
+            ::
+
+                    config = {
+                        "longitude": {"variable": "collection::group::variable"},
+                        "latitude": {"variable": "collection::group::variable"},
+                        "data": {"variable": "collection::group::variable",
+                                 "channel": "channel_name"},
+                        "plot_property": "property_value",
+                        "plot_option": "option_value",
+                        "schema": "path_to_schema_file.yaml"
+                    }
+                    logger = Logger()
+                    map_scatter_plot = MapScatter(config, logger, None)
+        """
+        self.config = config
+        self.logger = logger
+        self.dataobj = dataobj
+        self.xdata = []
+        self.ydata = []
+        self.plotobj = None
+
+# --------------------------------------------------------------------------------------------------
+
+    def data_prep(self):
+
+        # Get the data to plot from the data_collection
+        # ---------------------------------------------
+        var0 = self.config['x']['variable']
+        var1 = self.config['y']['variable']
+
+        var0_cgv = var0.split('::')
+        var1_cgv = var1.split('::')
+
+        if len(var0_cgv) != 3:
+            self.logger.abort('Scatter: comparison first var \'var0\' does not appear to ' +
+                              'be in the required format of collection::group::variable.')
+        if len(var1_cgv) != 3:
+            self.logger.abort('Scatter: comparison first var \'var1\' does not appear to ' +
+                              'be in the required format of collection::group::variable.')
+
+        # Optionally get the channel to plot
+        channel = None
+        if 'channel' in self.config:
+            channel = self.config.get('channel')
+
+        xdata = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2], channel)
+        xdata1 = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2])
+        ydata = self.dataobj.get_variable_data(var1_cgv[0], var1_cgv[1], var1_cgv[2], channel)
+
+        # see if we need to slice data
+        xdata = slice_var_from_str(self.config['x'], xdata, self.logger)
+        ydata = slice_var_from_str(self.config['y'], ydata, self.logger)
+
+        # scatter data should be flattened
+        xdata = xdata.flatten()
+        ydata = ydata.flatten()
+
+        # Remove NaN values to enable regression
+        # --------------------------------------
+        mask = ~np.isnan(xdata)
+        xdata = xdata[mask]
+        ydata = ydata[mask]
+
+        mask = ~np.isnan(ydata)
+        self.xdata = xdata[mask]
+        self.ydata = ydata[mask]
+
+    @abstractmethod
+    def configure_plot(self):
+        pass
+
+# --------------------------------------------------------------------------------------------------

src/eva/plotting/emcpy/plot_tools/figure_driver.py → src/eva/plotting/batch/base/plot_tools/figure_driver.py

@@ -14,7 +14,6 @@
 from eva.utilities.stats import stats_helper
 from eva.utilities.utils import get_schema, camelcase_to_underscore, parse_channel_list
 from eva.utilities.utils import replace_vars_dict
-from emcpy.plots.create_plots import CreatePlot, CreateFigure
 import copy
 import importlib as im
 import os
@@ -40,7 +39,28 @@ def figure_driver(config, data_collections, timing, logger):
 
     # Get list of graphics from configuration
     # -------------------
-    graphics = config.get("graphics")
+    graphics_section = config.get('graphics')
+    graphics = graphics_section.get('figure_list')
+
+    # Get plotting backend
+    # --------------------
+    backend = graphics_section.get('plotting_backend')
+
+    if backend == 'Hvplot':
+        try:
+            import hvplot
+        except ImportError:
+            logger.abort("The hvplot backend is not available since hvplot is not in the environment.")
+            backend = 'Emcpy'
+
+    # Create handler
+    # --------------
+    handler_class_name = backend + 'FigureHandler'
+    handler_module_name = camelcase_to_underscore(handler_class_name)
+    handler_full_module = 'eva.plotting.batch.' + \
+                          backend.lower() + '.plot_tools.' + handler_module_name
+    handler_class = getattr(im.import_module(handler_full_module), handler_class_name)
+    handler = handler_class()
 
     # Loop through specified graphics
     # -------------------
@@ -56,8 +76,8 @@ def figure_driver(config, data_collections, timing, logger):
 
         # update figure conf based on schema
         # ----------------------------------
-        fig_schema = figure_conf.get('schema', os.path.join(return_eva_path(), 'plotting',
-                                                            'emcpy', 'defaults', 'figure.yaml'))
+        fig_schema = figure_conf.get('schema', os.path.join(return_eva_path(), 'plotting', 'batch',
+                                     backend.lower(), 'defaults', 'figure.yaml'))
         figure_conf = get_schema(fig_schema, figure_conf, logger)
 
         # pass configurations and make graphic(s)
@@ -111,19 +131,20 @@ def figure_driver(config, data_collections, timing, logger):
                                                                   **batch_conf_this)
 
                     # Make plot
-                    make_figure(figure_conf_fill, plots_conf_fill,
+                    make_figure(handler, figure_conf_fill, plots_conf_fill,
                                 dynamic_options_conf_fill, data_collections, logger)
 
         else:
             # make just one figure per configuration
-            make_figure(figure_conf, plots_conf, dynamic_options_conf, data_collections, logger)
+            make_figure(handler, figure_conf, plots_conf,
+                        dynamic_options_conf, data_collections, logger)
     timing.stop('Graphics Loop')
 
 
 # --------------------------------------------------------------------------------------------------
 
 
-def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
+def make_figure(handler, figure_conf, plots, dynamic_options, data_collections, logger):
     """
     Generates a figure based on the provided configuration and plots.
 
@@ -143,7 +164,8 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
     # Adjust the plots configs if there are dynamic options
     # -----------------------------------------------------
     for dynamic_option in dynamic_options:
-        dynamic_option_module = im.import_module("eva.plotting.emcpy.plot_tools.dynamic_config")
+        mod_name = "eva.plotting.batch.base.plot_tools.dynamic_config"
+        dynamic_option_module = im.import_module(mod_name)
         dynamic_option_method = getattr(dynamic_option_module, dynamic_option['type'])
         plots = dynamic_option_method(logger, dynamic_option, plots, data_collections)
 
@@ -158,12 +180,23 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
     for plot in plots:
         layer_list = []
         for layer in plot.get("layers"):
-            eva_class_name = layer.get("type")
-            eva_module_name = camelcase_to_underscore(eva_class_name)
-            full_module = "eva.plotting.emcpy.diagnostics."+eva_module_name
-            layer_class = getattr(im.import_module(full_module), eva_class_name)
-            # use the translator class to go from eva to declarative plotting
-            layer_list.append(layer_class(layer, logger, data_collections).plotobj)
+
+            # Temporary case to handle different diagnostics
+            if handler.BACKEND_NAME == 'Emcpy':
+                eva_class_name = layer.get("type")
+                eva_module_name = camelcase_to_underscore(eva_class_name)
+                full_module = "eva.plotting.batch.emcpy.diagnostics."+eva_module_name
+                layer_class = getattr(im.import_module(full_module), eva_class_name)
+                layer_list.append(layer_class(layer, logger, data_collections).plotobj)
+            else:
+                eva_class_name = handler.BACKEND_NAME + layer.get("type")
+                eva_module_name = camelcase_to_underscore(eva_class_name)
+                full_module = handler.MODULE_NAME + eva_module_name
+                layer_class = getattr(im.import_module(full_module), eva_class_name)
+                layer = layer_class(layer, logger, data_collections)
+                layer.data_prep()
+                layer_list.append(layer.configure_plot())
+
         # get mapping dictionary
         proj = None
         domain = None
@@ -174,7 +207,7 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
             domain = mapoptions['domain']
 
         # create a subplot based on specified layers
-        plotobj = CreatePlot(plot_layers=layer_list, projection=proj, domain=domain)
+        plotobj = handler.create_plot(layer_list, proj, domain)
         # make changes to subplot based on YAML configuration
         for key, value in plot.items():
             if key not in ['layers', 'mapping', 'statistics']:
@@ -191,9 +224,10 @@ def make_figure(figure_conf, plots, dynamic_options, data_collections, logger):
         plot_list.append(plotobj)
 
     # create figure
-    fig = CreateFigure(nrows=figure_conf['layout'][0],
-                       ncols=figure_conf['layout'][1],
-                       figsize=tuple(figure_conf['figure size']))
+    nrows = figure_conf['layout'][0]
+    ncols = figure_conf['layout'][1]
+    figsize = tuple(figure_conf['figure size'])
+    fig = handler.create_figure(nrows, ncols, figsize)
     fig.plot_list = plot_list
     fig.create_figure()
 

src/eva/plotting/batch/emcpy/diagnostics/density.py

@@ -0,0 +1,91 @@
+from eva.eva_path import return_eva_path
+from eva.utilities.config import get
+from eva.utilities.utils import get_schema, update_object, slice_var_from_str
+import emcpy.plots.plots
+import os
+import numpy as np
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+class Density():
+
+    """Base class for creating density plots."""
+
+    def __init__(self, config, logger, dataobj):
+
+        """
+        Creates a density plot based on the provided configuration and data.
+
+        Args:
+            config (dict): A dictionary containing the configuration for the density plot.
+            logger (Logger): An instance of the logger for logging messages.
+            dataobj: An instance of the data object containing input data.
+
+        This class initializes and configures a density plot based on the provided configuration and
+        data. The density plot is created using a declarative plotting library from EMCPy
+        (https://github.com/NOAA-EMC/emcpy).
+
+        Example:
+
+            ::
+
+                    config = {
+                        "data": {
+                            "variable": "collection::group::variable",
+                            "channel": "channel_name",
+                            "slicing": "slice expression"
+                        },
+                        "plot_property": "property_value",
+                        "plot_option": "option_value",
+                        "schema": "path_to_schema_file.yaml"
+                    }
+                    logger = Logger()
+                    dataobj = DataObject()
+                    density_plot = Density(config, logger, dataobj)
+        """
+
+        # Get the data to plot from the data_collection
+        # ---------------------------------------------
+        varstr = config['data']['variable']
+        var_cgv = varstr.split('::')
+
+        if len(var_cgv) != 3:
+            logger.abort('In Density the variable \'var_cgv\' does not appear to ' +
+                         'be in the required format of collection::group::variable.')
+
+        # Optionally get the channel to plot
+        channel = None
+        if 'channel' in config['data']:
+            channel = config['data'].get('channel')
+
+        data = dataobj.get_variable_data(var_cgv[0], var_cgv[1], var_cgv[2], channel)
+
+        # See if we need to slice data
+        data = slice_var_from_str(config['data'], data, logger)
+
+        # Density data should be flattened
+        data = data.flatten()
+
+        # Missing data should also be removed
+        mask = ~np.isnan(data)
+        data = data[mask]
+
+        # Create declarative plotting density object
+        # --------------------------------------------
+        self.plotobj = emcpy.plots.plots.Density(data)
+
+        # Get defaults from schema
+        # ------------------------
+        layer_schema = config.get('schema', os.path.join(return_eva_path(), 'plotting',
+                                                         'batch', 'emcpy', 'defaults',
+                                                         'density.yaml'))
+        config = get_schema(layer_schema, config, logger)
+        delvars = ['type', 'schema', 'data']
+        for d in delvars:
+            config.pop(d, None)
+        self.plotobj = update_object(self.plotobj, config, logger)
+
+
+# --------------------------------------------------------------------------------------------------