Update TEM configuration

config_amwg_default_plots.yaml

@@ -182,6 +182,20 @@ diag_cam_climo:
     #Location where time series files are (or will be) stored:
     cam_ts_loc: /glade/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts
 
+    #TEM diagnostics
+    #---------------
+    #TEM history file number
+    #If missing or blank, ADF will default to h4
+    tem_hist_str: cam.h4
+
+    #Location where TEM files are stored:
+    #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped!
+    cam_tem_loc: /glade/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/
+
+    #Overwrite TEM files, if found?
+    #If set to false, then TEM creation will be skipped if files are found:
+    overwrite_tem: false
+
     #----------------------
 
 #This third set of variables provide info for the CAM baseline climatologies.
@@ -239,6 +253,20 @@ diag_cam_baseline_climo:
     #Location where time series files are (or will be) stored:
     cam_ts_loc: /glade/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts
 
+    #TEM diagnostics
+    #---------------
+    #TEM history file number
+    #If missing or blank, ADF will default to h4
+    tem_hist_str: cam.h4
+
+    #Location where TEM files are stored:
+    #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped!
+    cam_tem_loc: /glade/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/
+
+    #Overwrite TEM files, if found?
+    #If set to false, then TEM creation will be skipped if files are found:
+    overwrite_tem: false
+
 #This fourth set of variables provides settings for calling the Climate Variability
 # Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and
 # run in background mode, likely completing after the ADF has completed.
@@ -298,8 +326,7 @@ plotting_scripts:
     - polar_map
     - cam_taylor_diagram
     #- tape_recorder
-    #- tem #To plot TEM, please un-comment fill-out
-                              #the "tem_info" section below
+    #- tem
     #- regional_map_multicase #To use this please un-comment and fill-out
                               #the "region_multicase" section below
 
@@ -361,25 +388,4 @@ diag_var_list:
 #     region_season: <NULL means use annual mean>
 #     region_variables: <list of variables to try to use; allows for a subset of the total diag variables>
 
-# Options for TEM diagnostics (./averaging/create_TEM_files.py and ./plotting/temp.py)
-#tem_info:
-    #Location where TEM files are stored:
-    #If path not specified or commented out, TEM calculation/plots will be skipped
-#    tem_loc: /glade/scratch/richling/adf-output/ADF-data/TEM/
-
-    #TEM history file number
-    #If missing or blank, ADF will default to h4
-#    hist_num: h4
-
-    #Overwrite TEM files, if found?
-    #If set to false, then TEM creation will be skipped if files are found:
-#    overwrite_tem_case: false
-
-    #For multi-case
-    #overwrite_tem_case:
-    #    - false
-    #    - true
-
-#    overwrite_tem_base: false
-
 #END OF FILE

config_cam_baseline_example.yaml

@@ -182,6 +182,20 @@ diag_cam_climo:
     #Location where time series files are (or will be) stored:
     cam_ts_loc: /glade/scratch/${user}/ADF/${diag_cam_climo.cam_case_name}/ts
 
+    #TEM diagnostics
+    #---------------
+    #TEM history file number
+    #If missing or blank, ADF will default to h4
+    tem_hist_str: cam.h4
+
+    #Location where TEM files are stored:
+    #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped!
+    cam_tem_loc: /glade/scratch/${user}/${diag_cam_climo.cam_case_name}/tem/
+
+    #Overwrite TEM files, if found?
+    #If set to false, then TEM creation will be skipped if files are found:
+    overwrite_tem: false
+
     #----------------------
 
     #You can alternatively provide a list of cases, which will make the ADF
@@ -245,6 +259,26 @@ diag_cam_climo:
     #    - /some/where/you/want/to/have/time_series_files
     #    - /same/or/different/place/you/want/files
 
+    #TEM diagnostics
+    #---------------
+    #TEM history file number
+    #If missing or blank, ADF will default to h4
+    #tem_hist_str:
+    #    - cam.h4
+    #    - cam.h#
+
+    #Location where TEM files are stored:
+    #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped!
+    #cam_tem_loc:
+    #    - /some/where/you/want/to/have/TEM_files/
+    #    - /same/or/different/place/you/want/TEM_files/
+
+    #Overwrite TEM files, if found?
+    #If set to false, then TEM creation will be skipped if files are found:
+    #overwrite_tem:
+    #    - false
+    #    - true
+
     #----------------------
 
 
@@ -303,6 +337,21 @@ diag_cam_baseline_climo:
     #Location where time series files are (or will be) stored:
     cam_ts_loc: /glade/scratch/${user}/ADF/${diag_cam_baseline_climo.cam_case_name}/ts
 
+    #TEM diagnostics
+    #---------------
+    #TEM history file number
+    #If missing or blank, ADF will default to h4
+    tem_hist_str: cam.h4
+
+    #Location where TEM files are stored:
+    #NOTE: If path not specified or commented out, TEM calculation/plots will be skipped!
+    cam_tem_loc: /glade/scratch/${user}/${diag_cam_baseline_climo.cam_case_name}/tem/
+
+    #Overwrite TEM files, if found?
+    #If set to false, then TEM creation will be skipped if files are found:
+    overwrite_tem: false
+
+
 #This fourth set of variables provides settings for calling the Climate Variability
 # Diagnostics Package (CVDP). If cvdp_run is set to true the CVDP will be set up and
 # run in background mode, likely completing after the ADF has completed.
@@ -364,8 +413,7 @@ plotting_scripts:
     - cam_taylor_diagram
     - qbo
     #- tape_recorder
-    #- tem #To plot TEM, please un-comment fill-out
-                              #the "tem_info" section below
+    #- tem
     #- regional_map_multicase #To use this please un-comment and fill-out
                               #the "region_multicase" section below
 
@@ -403,25 +451,4 @@ diag_var_list:
 #     region_season: <NULL means use annual mean>
 #     region_variables: <list of variables to try to use; allows for a subset of the total diag variables>
 
-# Options for TEM diagnostics (./averaging/create_TEM_files.py and ./plotting/temp.py)
-#tem_info:
-    #Location where TEM files are stored:
-    #If path not specified or commented out, TEM calculation/plots will be skipped
-#    tem_loc: /glade/scratch/richling/adf-output/ADF-data/TEM/
-
-    #TEM history file number
-    #If missing or blank, ADF will default to h4
-#    hist_num: h4
-
-    #Overwrite TEM files, if found?
-    #If set to false, then TEM creation will be skipped if files are found:
-#    overwrite_tem_case: false
-
-    #For multi-case
-    #overwrite_tem_case:
-    #    - false
-    #    - true
-
-#    overwrite_tem_base: false
-
 #END OF FILE

scripts/averaging/create_TEM_files.py

@@ -14,9 +14,13 @@ def create_TEM_files(adf):
 
     """
 
+    #Notify user that script has started:
+    print("\n  Generating CAM TEM diagnostics files...")
+
     #Special ADF variables
     #CAM simulation variables (these quantities are always lists):
     case_names    = adf.get_cam_info("cam_case_name", required=True)
+    base_name     = adf.get_baseline_info("cam_case_name")
 
     #Grab h4 history files locations
     cam_hist_locs = adf.get_cam_info("cam_hist_loc", required=True)
@@ -25,16 +29,37 @@ def create_TEM_files(adf):
     start_years   = adf.climo_yrs["syears"]
     end_years     = adf.climo_yrs["eyears"]
 
+
+    tem_locs = []
+
     #Grab TEM diagnostics options
-    tem_opts = adf.read_config_var("tem_info")
+    #----------------------------
+    #Extract TEM file save locations
+    tem_base_loc = adf.get_baseline_info("cam_tem_loc")
+    tem_case_locs = adf.get_cam_info("cam_tem_loc", required=True)
+
+    #If path not specified, skip TEM calculation?
+    if tem_case_locs is None:
+        print("\t 'cam_tem_loc' not found in 'diag_cam_climo', so no TEM files/diagnostics will be generated.")
+        pass
+    else:
+        for tem_case_loc in tem_case_locs:
+            tem_case_loc = Path(tem_case_loc)
+            #Check if TEM directory exists, and if not, then create it:
+            if not tem_case_loc.is_dir():
+                print(f"    {tem_case_loc} not found, making new directory")
+                tem_case_loc.mkdir(parents=True)
+            #End if
+            tem_locs.append(tem_case_loc)
+        #End for
 
-    if not tem_opts:
-        print("\n  No TEM options provided, skipping TEM file creation." \
-        "\nSee documentation or config_cam_baseline_example.yaml for options to add to configuration file.")
-        return
+    #Set default to h4
+    hist_nums = adf.get_cam_info("tem_hist_str")
+    if hist_nums is None:
+        hist_nums = ["h4"]*len(case_names)
 
     #Get test case(s) tem over-write boolean and force to list if not by default
-    overwrite_tem_cases = tem_opts.get("overwrite_tem_case")
+    overwrite_tem_cases = adf.get_cam_info("overwrite_tem")
 
     #If overwrite argument is missing, then default to False:
     if overwrite_tem_cases is None:
@@ -50,45 +75,60 @@ def create_TEM_files(adf):
         #If dictionary is empty, then there are no observations, so quit here:
         if not var_obs_dict:
             print("No observations found to plot against, so no TEM will be generated.")
-            return
+            pass
 
         base_name = "Obs"
-    else:
-        base_name = adf.get_baseline_info("cam_case_name", required=True)
-        cam_hist_locs.append(adf.get_baseline_info("cam_hist_loc", required=True))
 
-        #Extract baseline years (which may be empty strings if using Obs):
-        syear_baseline = adf.climo_yrs["syear_baseline"]
-        eyear_baseline = adf.climo_yrs["eyear_baseline"]
-
-        case_names.append(base_name)
-        start_years.append(syear_baseline)
-        end_years.append(eyear_baseline)
-        overwrite_tem_cases.append(tem_opts.get("overwrite_tem_base", False))
-    #End if
+        tem_base_loc = adf.get_basic_info('obs_tem_loc')
+
+        #If path not specified, skip TEM calculation?
+        if tem_base_loc is None:
+            print("\t 'obs_tem_loc' not found in config file, so no TEM files/diagnostics will be generated.")
+            return
+        else:
+            tem_base_loc = Path(tem_base_loc)
+            #Check if TEM directory exists, and if not, then create it:
+            if not tem_base_loc.is_dir():
+                print(f"    {tem_base_loc} not found, making new directory")
+                tem_base_loc.mkdir(parents=True)
+            #End if
 
-    #Set default to h4
-    hist_num = tem_opts.get("hist_num")
-    if hist_num is None:
-        hist_num = "h4"
+    else:
+        if tem_base_loc:
+            cam_hist_locs.append(adf.get_baseline_info("cam_hist_loc", required=True))
+
+            #Set default to h4
+            hist_num = adf.get_baseline_info("tem_hist_str")
+            if hist_num is None:
+                hist_num = "h4"
+
+            #Extract baseline years (which may be empty strings if using Obs):
+            syear_baseline = adf.climo_yrs["syear_baseline"]
+            eyear_baseline = adf.climo_yrs["eyear_baseline"]
+
+            case_names.append(base_name)
+            start_years.append(syear_baseline)
+            end_years.append(eyear_baseline)
+
+            tem_base_loc = Path(tem_base_loc)
+            #Check if TEM directory exists, and if not, then create it:
+            if not tem_base_loc.is_dir():
+                print(f"    {tem_base_loc} not found, making new directory")
+                tem_base_loc.mkdir(parents=True)
+            #End if
 
-    #Extract TEM file save location
-    output_loc = tem_opts["tem_loc"]
+            tem_locs.append(tem_base_loc)
+            overwrite_tem_cases.append(adf.get_baseline_info("overwrite_tem", False))
 
-    #If path not specified, skip TEM calculation?
-    if output_loc is None:
-        print("\t 'tem_loc' not found in config file, so no TEM files will be generated.")
-        return
-    else:
-        #Notify user that script has started:
-        print("\n  Generating CAM TEM diagnostics files...")
+            #Set default to h4
+            hist_num = adf.get_baseline_info("hist_num")
+            if hist_num is None:
+                hist_num = "h4"
+            hist_nums.append(hist_num)
+        else:
+            print("\t 'cam_tem_loc' not found in 'diag_cam_baseline_climo', so no TEM files/diagnostics will be generated.")
 
-        output_loc = Path(output_loc)
-        #Check if re-gridded directory exists, and if not, then create it:
-        if not output_loc.is_dir():
-            print(f"    {output_loc} not found, making new directory")
-            output_loc.mkdir(parents=True)
-        #End if
+    #End if
 
     res = adf.variable_defaults # will be dict of variable-specific plot preferences
 
@@ -102,18 +142,13 @@ def create_TEM_files(adf):
     if adf.get_basic_info("compare_obs"):
         print(f"\t Processing TEM for observations :")
 
-        output_loc_idx = output_loc / base_name
-        #Check if re-gridded directory exists, and if not, then create it:
-        if not output_loc_idx.is_dir():
-            print(f"    {output_loc_idx} not found, making new directory")
-            output_loc_idx.mkdir(parents=True)
-        #End if
-
         #Set baseline file name as full path
-        tem_fil = output_loc_idx / f'{base_name}.TEMdiag.nc'
+        tem_fil = tem_base_loc / f'{base_name}.TEMdiag.nc'
 
-        #Get baseline case tem over-write boolean
-        overwrite_tem = tem_opts.get("overwrite_tem_base")
+        #Get obs case tem over-write boolean
+        #QUESTION: should we make these and put in ADF obs location so we don't have to 
+        #worry about making these??
+        overwrite_tem = False
 
         #If files exist, then check if over-writing is allowed:
         if (tem_fil.is_file()) and (not overwrite_tem):
@@ -189,15 +224,16 @@ def create_TEM_files(adf):
         #End if
 
         #Check if history files actually exist. If not then kill script:
-        hist_str = '*.cam.'+hist_num
+        #hist_str = '*.cam.'+hist_nums[case_idx]
+        hist_str = f"*{hist_nums[case_idx]}"
         if not list(starting_location.glob(hist_str+'.*.nc')):
             emsg = f"No CAM history {hist_str} files found in '{starting_location}'."
             emsg += " Script is ending here."
             adf.end_diag_fail(emsg)
         #End if
 
         #Get full path and file for file name
-        output_loc_idx = output_loc / case_name
+        output_loc_idx = tem_locs[case_idx]
 
         #Check if re-gridded directory exists, and if not, then create it:
         if not output_loc_idx.is_dir():
@@ -459,4 +495,4 @@ def calc_tem(ds):
                                       utendwtem = utendwtem
                                       ))
 
-    return dstem
+    return dstem