Merge pull request #511 from wrongkindofdoctor/refactor_pp

More updates for refactor_pp branch

diagnostics/example_multicase/multirun_config_demo1.jsonc

@@ -24,7 +24,7 @@
   // If a relative path is given, it's resolved relative to the MDTF-diagnostics
   // code directory. Environment variables (eg, $HOME) can be referenced with a
   // "$" and will be expended to their current values when the framework runs.
-  // Full path to model data ESM-intake catalog header file
+  // Full or relative path to model data ESM-intake catalog header file
   "DATA_CATALOG":"/home/a1r/github/MDTF-diagnostics/diagnostics/example_multicase/c384L65_am5f3b1r0_amip.json",
 
   // Backwards compatibility

diagnostics/example_multicase/multirun_config_demo2.jsonc

@@ -24,7 +24,7 @@
   // If a relative path is given, it's resolved relative to the MDTF-diagnostics
   // code directory. Environment variables (eg, $HOME) can be referenced with a
   // "$" and will be expended to their current values when the framework runs.
-  // Full path to model data ESM-intake catalog header file
+  // Full or relative path to model data ESM-intake catalog header file
   "DATA_CATALOG": "/home/a1r/github/aparna/MDTF-diagnostics/diagnostics/example_multicase/amip_c96L65_am5f3b1r0_pdclim1850F_combined.json",
 
   // Backwards compatibility

diagnostics/example_multicase/multirun_config_template.jsonc

@@ -49,8 +49,8 @@
   // If a relative path is given, it's resolved relative to the MDTF-diagnostics
   // code directory. Environment variables (eg, $HOME) can be referenced with a
   // "$" and will be expended to their current values when the framework runs.
-  // Full path to model data ESM-intake catalog header file
-  "DATA_CATALOG": "/net/jml/mdtf/MDTF-diagnostics/diagnostics/example_multicase/esm_catalog_CMIP_synthetic_r1i1p1f1_gr1.json",
+  // Full or relative path to model data ESM-intake catalog header file
+  "DATA_CATALOG": "../diagnostics/example_multicase/esm_catalog_CMIP_synthetic_r1i1p1f1_gr1.json",
 
   // Backwards compatibility
   "MODEL_DATA_ROOT": "../inputdata/mdtf_test_data",

mdtf_framework.py

@@ -139,6 +139,8 @@ def backup_config(config):
     model_paths.setup_data_paths(ctx.config.case_list)
     ctx.config.update({'WORK_DIR': model_paths.WORK_DIR})
     ctx.config.update({'OUTPUT_DIR': model_paths.OUTPUT_DIR})
+    cat_path = ctx.config.DATA_CATALOG
+    ctx.config.update({'DATA_CATALOG': util.filesystem.resolve_path(cat_path)})
     # TODO: update paths in ctx.config so that POD paths are placed in the correct sub-directories
     backup_config = backup_config(ctx.config)
     ctx.config._configs = dict()

src/util/filesystem.py

@@ -18,7 +18,7 @@
 _log = logging.getLogger(__name__)
 
 
-def abbreviate_path(path, old_base, new_base=None):
+def abbreviate_path(path: str, old_base: str, new_base=None) -> str:
     """Express *path* as a path relative to *old_base*, optionally prepending
     *new_base*.
     """
@@ -29,11 +29,12 @@ def abbreviate_path(path, old_base, new_base=None):
     return str_
 
 
-def resolve_path(rel_path: str, root_path: str = "", env_vars: dict = None, log=_log):
+def resolve_path(rel_path: str, root_path: str = "", env_vars: dict = None, log=_log) -> str:
     """Abbreviation to resolve relative paths, expanding environment variables
     if necessary.
 
     Args:
+        log: logger object
         rel_path (str): Path to resolve.
         root_path (str): Optional. Root path to resolve `path` with. If
             not given, resolves relative to :py:func:`os.getcwd`.
@@ -69,12 +70,12 @@ def _expandvars(path_name: str, env_dict: dict):
         return rel_path
     if root_path == "":
         root_path = os.getcwd()
-    assert os.path.isabs(root_path)
+    assert os.path.isabs(root_path), f"{root_path} is not an absolute path"
     return os.path.normpath(os.path.join(root_path, rel_path))
 
 
-def recursive_copy(src_files, src_root, dest_root, copy_function=None,
-                   overwrite=False):
+def recursive_copy(src_files, src_root: str, dest_root: str, copy_function=None,
+                   overwrite: bool = False):
     """Copy *src_files* to *dest_root*, preserving relative subdirectory structure.
 
     Copies a subset of files in a directory subtree rooted at *src_root* to an
@@ -118,7 +119,7 @@ def recursive_copy(src_files, src_root, dest_root, copy_function=None,
         copy_function(src, dest)
 
 
-def check_executable(exec_name):
+def check_executable(exec_name: str) -> bool:
     """Tests if the executable *exec_name* is found on the current ``$PATH``.
 
     Args:
@@ -127,7 +128,7 @@ def check_executable(exec_name):
     return find_executable(exec_name) is not None
 
 
-def find_files(src_dirs, filename_globs, n_files=None):
+def find_files(src_dirs: str | list, filename_globs: str | list, n_files=None) -> list:
     """Return list of files in *src_dirs*, or any subdirectories, matching any
     of *filename_globs*. Wraps Python :py:class:`glob.glob`.
 
@@ -160,7 +161,7 @@ def find_files(src_dirs, filename_globs, n_files=None):
     return list(files)
 
 
-def check_dir(dir_path, attr_name="", create=False):
+def check_dir(dir_path: str, attr_name: str = "", create: bool = False):
     """Check existence of directories. No action is taken for directories that
     already exist; nonexistent directories either raise a
     :class:`~util.MDTFFileNotFoundError` or cause the creation of that directory.
@@ -199,7 +200,7 @@ def check_dir(dir_path, attr_name="", create=False):
                 from exc
 
 
-def bump_version(path, new_v=None, extra_dirs=None):
+def bump_version(path: str, new_v=None, extra_dirs=None):
     """Append a version number to *path*, if necessary, so that it doesn't
     conflict with existing files.
 
@@ -307,8 +308,8 @@ class _DoubleBraceTemplate(string.Template):
     """
 
 
-def append_html_template(template_file, target_file, template_dict={},
-                         create=True, append=True):
+def append_html_template(template_file: str, target_file: str, template_dict: dict={},
+                         create: bool = True, append: bool = True):
     """Perform substitutions on *template_file* and write result to *target_file*.
 
     Variable substitutions are done with custom
@@ -400,7 +401,7 @@ def make_tempdir(self, hash_obj=None):
         self._dirs.append(new_dir)
         return new_dir
 
-    def rm_tempdir(self, path):
+    def rm_tempdir(self, path: str):
         assert path in self._dirs
         self._dirs.remove(path)
         _log.debug("Cleaning up temp dir %s", path)

tests/esm_catalog_CESM_test_macos.json

@@ -0,0 +1,191 @@
+{
+  "esmcat_version": "0.0.1",
+  "attributes": [
+    {
+      "column_name": "activity_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "branch_method",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "branch_time_in_child",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "branch_time_in_parent",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "experiment",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "experiment_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "frequency",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "grid",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "grid_label",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "institution_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "nominal_resolution",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "parent_activity_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "parent_experiment_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "parent_source_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "parent_time_units",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "parent_variant_label",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "product",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "realm",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "source_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "source_type",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "sub_experiment",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "sub_experiment_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "table_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variable_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "variant_label",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "member_id",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "standard_name",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "long_name",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "units",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "vertical_levels",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "init_year",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "start_time",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "end_time",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "time_range",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "path",
+      "vocabulary": ""
+    },
+    {
+      "column_name": "version",
+      "vocabulary": ""
+    }
+  ],
+  "assets": {
+    "column_name": "path",
+    "format": "netcdf",
+    "format_column_name": null
+  },
+  "aggregation_control": {
+    "variable_column_name": "variable_id",
+    "groupby_attrs": [
+      "activity_id",
+      "institution_id",
+      "source_id",
+      "experiment_id",
+      "frequency",
+      "member_id",
+      "table_id",
+      "grid_label",
+      "realm",
+      "variant_label",
+      "time_range"
+    ],
+    "aggregations": [
+      {
+        "type": "union",
+        "attribute_name": "variable_id",
+        "options": {}
+      },
+      {
+        "type": "join_existing",
+        "attribute_name": "time_range",
+        "options": {
+          "dim": "time",
+          "coords": "minimal",
+          "compat": "override"
+        }
+      }
+    ]
+  },
+  "id": "esm_catalog_CESM_test_macos.csv",
+  "description": null,
+  "title": null,
+  "last_updated": "2023-06-01",
+  "catalog_file": "file:/Users/runner/mdtf/MDTF-diagnostics/tests/esm_catalog_CESM_test_macos.csv"
+}