From f953ef35d031d3f098c5510df0cf9132ab997a6f Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Fri, 22 Sep 2023 15:46:41 -0600 Subject: [PATCH 1/8] Adding mods to derive PRECT from PRECC+PRECL if PRECT is not in the cam.ho files. This code could be extended to other variables. --- lib/adf_diag.py | 49 ++++++++++++++++++++++++++++++---- lib/adf_variable_defaults.yaml | 7 +++++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 88d34f32f..9061c39c2 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -379,6 +379,9 @@ def call_ncrcat(cmd): hist_str = 'cam.h0' #End if + # get info about variables defaults + res = self.variable_defaults + #Loop over cases: for case_idx, case_name in enumerate(case_names): @@ -510,12 +513,24 @@ def call_ncrcat(cmd): #Loop over CAM history variables: list_of_commands = [] - for var in self.diag_var_list: + vars_to_derive = [] + #create copy of var list that can be modified for derivable variables + diag_var_list = self.diag_var_list + for var_index,var in enumerate(diag_var_list): if var not in hist_file_var_list: - msg = f"WARNING: {var} is not in the file {hist_files[0]}." - msg += " No time series will be generated." - print(msg) - continue + vres = res[var] + if "derivable_from" in vres: + constit_list = vres['derivable_from'] + for constit in constit_list: + if constit not in diag_var_list: + diag_var_list.append(constit) + vars_to_derive.append(var) + continue + else: + msg = f"WARNING: {var} is not in the file {hist_files[0]}." + msg += " No time series will be generated." + print(msg) + continue #Check if variable has a "lev" dimension according to first file: has_lev = bool('lev' in hist_file_ds[var].dims) @@ -592,6 +607,9 @@ def call_ncrcat(cmd): #Now run the "ncrcat" subprocesses in parallel: with mp.Pool(processes=self.num_procs) as mpool: _ = mpool.map(call_ncrcat, list_of_commands) + + if vars_to_derive: + self.derive_variables(vars_to_derive=vars_to_derive,directory=ts_dir[case_idx]) #End with #End cases loop @@ -915,4 +933,25 @@ def setup_run_cvdp(self): print('For CVDP information visit: https://www.cesm.ucar.edu/working_groups/CVC/cvdp/') print(' ') + ######### + + def derive_variables(self,vars_to_derive=None,directory=None): + + """ + Derive variables acccording to steps given here. Since derivations will depend on the + variable, each variable to derive will need its own set of steps below. + """ + + for var in vars_to_derive: + if var == "PRECT": + # PRECT can be found by simply adding PRECL and PRECC + # grab file names for the PRECL and PRECC files from the case ts directory + constit_files = sorted(glob.glob(directory+"/*PREC*")) + # create new file name for PRECT + prect_file = constit_files[0].replace('PRECC','PRECT') + # append PRECC to the file containing PRECL + os.system(f"ncks -A -v PRECC {constit_files[0]} {constit_files[1]}") + # create new file with the sum of PRECC and PRECL + os.system(f"ncap2 -s 'PRECT=(PRECC+PRECL)' {constit_files[1]} {prect_file}") + ############### diff --git a/lib/adf_variable_defaults.yaml b/lib/adf_variable_defaults.yaml index af554f4d6..7f6bf92c5 100644 --- a/lib/adf_variable_defaults.yaml +++ b/lib/adf_variable_defaults.yaml @@ -49,6 +49,12 @@ # category -> The website category the variable will be placed under. # # +# DERIVING: +# +# derivable_from -> If not present in the available output files, the variable can be derived from +# other variables that are present (e.g. PRECT can be derived from PRECC and PRECL), +# which are specified in this list +# # Final Note: Please do not modify this file unless you plan to push your changes back to the ADF repo. # If you would like to modify this file for your personal ADF runs then it is recommended # to make a copy of this file, make modifications in that copy, and then point the ADF to @@ -445,6 +451,7 @@ PRECT: obs_name: "ERAI" obs_var_name: "PRECT" category: "Hydrologic cycle" + derivable_from: ['PRECL','PRECC'] QFLX: category: "Hydrologic cycle" From 8041bc7f4d096235e68f39840649cbb97e8dacc3 Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Fri, 22 Sep 2023 15:50:58 -0600 Subject: [PATCH 2/8] Removing unncessary use of 'enumerate'. --- lib/adf_diag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 9061c39c2..3a01e94bb 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -516,7 +516,7 @@ def call_ncrcat(cmd): vars_to_derive = [] #create copy of var list that can be modified for derivable variables diag_var_list = self.diag_var_list - for var_index,var in enumerate(diag_var_list): + for var in diag_var_list: if var not in hist_file_var_list: vres = res[var] if "derivable_from" in vres: From 0fb6a787b926236d6d9bd90d5a59874b4d9e57d4 Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Fri, 22 Sep 2023 15:53:34 -0600 Subject: [PATCH 3/8] Renamed directory for clarity. --- lib/adf_diag.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 3a01e94bb..577a52d98 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -609,7 +609,7 @@ def call_ncrcat(cmd): _ = mpool.map(call_ncrcat, list_of_commands) if vars_to_derive: - self.derive_variables(vars_to_derive=vars_to_derive,directory=ts_dir[case_idx]) + self.derive_variables(vars_to_derive=vars_to_derive,ts_dir=ts_dir[case_idx]) #End with #End cases loop @@ -935,7 +935,7 @@ def setup_run_cvdp(self): ######### - def derive_variables(self,vars_to_derive=None,directory=None): + def derive_variables(self,vars_to_derive=None,ts_dir=None): """ Derive variables acccording to steps given here. Since derivations will depend on the From 653c3935552ffdb0bfe721af4bfa1812dac4b0ff Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Sat, 23 Sep 2023 08:12:09 -0600 Subject: [PATCH 4/8] Fixing directory variable name. --- lib/adf_diag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 577a52d98..26f2a1ad9 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -946,7 +946,7 @@ def derive_variables(self,vars_to_derive=None,ts_dir=None): if var == "PRECT": # PRECT can be found by simply adding PRECL and PRECC # grab file names for the PRECL and PRECC files from the case ts directory - constit_files = sorted(glob.glob(directory+"/*PREC*")) + constit_files = sorted(glob.glob(ts_dir+"/*PREC*")) # create new file name for PRECT prect_file = constit_files[0].replace('PRECC','PRECT') # append PRECC to the file containing PRECL From 33bf04fbd0ec422ffe89f268ef057e825fc268c3 Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Fri, 20 Oct 2023 08:50:40 -0600 Subject: [PATCH 5/8] Updates for PR change requests. --- lib/adf_diag.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 26f2a1ad9..baa0b7906 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -379,7 +379,7 @@ def call_ncrcat(cmd): hist_str = 'cam.h0' #End if - # get info about variables defaults + # get info about variable defaults res = self.variable_defaults #Loop over cases: @@ -518,7 +518,7 @@ def call_ncrcat(cmd): diag_var_list = self.diag_var_list for var in diag_var_list: if var not in hist_file_var_list: - vres = res[var] + vres = res.get(var, {}) if "derivable_from" in vres: constit_list = vres['derivable_from'] for constit in constit_list: @@ -946,7 +946,11 @@ def derive_variables(self,vars_to_derive=None,ts_dir=None): if var == "PRECT": # PRECT can be found by simply adding PRECL and PRECC # grab file names for the PRECL and PRECC files from the case ts directory - constit_files = sorted(glob.glob(ts_dir+"/*PREC*")) + if len(glob.glob(os.path.join(ts_dir,"*PRECC*"))) and len(glob.glob(os.path.join(ts_dir,"*PRECL*"))): + constit_files=sorted(glob.glob(os.path.join(ts_dir,"*PREC*"))) + else: + ermsg = "PRECC and PRECL were not both present; PRECT cannot be calculated." + raise FileNotFoundError(ermsg) # create new file name for PRECT prect_file = constit_files[0].replace('PRECC','PRECT') # append PRECC to the file containing PRECL From 5ddfc4332d896c94fb61eca46086b7e3f92f31df Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Fri, 20 Oct 2023 09:50:17 -0600 Subject: [PATCH 6/8] Updating PR for changes requested. --- lib/adf_diag.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index baa0b7906..8ac32a297 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -946,10 +946,16 @@ def derive_variables(self,vars_to_derive=None,ts_dir=None): if var == "PRECT": # PRECT can be found by simply adding PRECL and PRECC # grab file names for the PRECL and PRECC files from the case ts directory - if len(glob.glob(os.path.join(ts_dir,"*PRECC*"))) and len(glob.glob(os.path.join(ts_dir,"*PRECL*"))): +# if len(glob.glob(os.path.join(ts_dir,"*PRECC*"))) and len(glob.glob(os.path.join(ts_dir,"*PRECL*"))): +# constit_files=sorted(glob.glob(os.path.join(ts_dir,"*PREC*"))) +# else: +# ermsg = "PRECC and PRECL were not both present; PRECT cannot be calculated." +# raise FileNotFoundError(ermsg) + if glob.glob(os.path.join(ts_dir,"*PRECC*")) and glob.glob(os.path.join(ts_dir,"*PRECL*")): constit_files=sorted(glob.glob(os.path.join(ts_dir,"*PREC*"))) else: ermsg = "PRECC and PRECL were not both present; PRECT cannot be calculated." + ermsg += " Please remove PRECT from diag_var_list or find the relevant CAM files." raise FileNotFoundError(ermsg) # create new file name for PRECT prect_file = constit_files[0].replace('PRECC','PRECT') From ffc7de47fbc687e87e4094350d4847013cf32591 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Fri, 20 Oct 2023 10:10:40 -0600 Subject: [PATCH 7/8] Replace deprecated 'do_exit' flag with 'exit' in pylint script. --- .github/scripts/pylint_threshold_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/pylint_threshold_test.py b/.github/scripts/pylint_threshold_test.py index 3cefc0d33..8e61d8035 100755 --- a/.github/scripts/pylint_threshold_test.py +++ b/.github/scripts/pylint_threshold_test.py @@ -87,7 +87,7 @@ def pylint_check(pyfile_list, rcfile, threshold=10.0): #Run linter: lint_results = lint.Run([rcstr, '--exit-zero', pyfile], - reporter=pylint_report, do_exit=False) + reporter=pylint_report, exit=False) #Extract linter score: lint_score = lint_results.linter.stats.global_note From b1104566b9eda8f404b455fdda35e4dcb0aff586 Mon Sep 17 00:00:00 2001 From: bstephens82 Date: Fri, 20 Oct 2023 12:18:33 -0600 Subject: [PATCH 8/8] Removing old lines. --- lib/adf_diag.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/adf_diag.py b/lib/adf_diag.py index 8ac32a297..50558824b 100644 --- a/lib/adf_diag.py +++ b/lib/adf_diag.py @@ -946,11 +946,6 @@ def derive_variables(self,vars_to_derive=None,ts_dir=None): if var == "PRECT": # PRECT can be found by simply adding PRECL and PRECC # grab file names for the PRECL and PRECC files from the case ts directory -# if len(glob.glob(os.path.join(ts_dir,"*PRECC*"))) and len(glob.glob(os.path.join(ts_dir,"*PRECL*"))): -# constit_files=sorted(glob.glob(os.path.join(ts_dir,"*PREC*"))) -# else: -# ermsg = "PRECC and PRECL were not both present; PRECT cannot be calculated." -# raise FileNotFoundError(ermsg) if glob.glob(os.path.join(ts_dir,"*PRECC*")) and glob.glob(os.path.join(ts_dir,"*PRECL*")): constit_files=sorted(glob.glob(os.path.join(ts_dir,"*PREC*"))) else: