From e65162f3083c34b639854e3216e2355ff6fccc0b Mon Sep 17 00:00:00 2001
From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com>
Date: Fri, 29 Sep 2023 19:18:07 -0400
Subject: [PATCH] Update module loads to intel/2022 for jobs running gsi.x and
 enkf.x (#1882)

This PR changes the module loads in g-w jobs `anal.sh`, `eobs.sh`, and `eupd.sh`
to use machine specific modulefiles from `sorc/gsi_enkf.fd/modulefiles`.  This is
necessary because [NOAA-EMC/GSI#571](https://github.com/NOAA-EMC/GSI/pull/571)
updated `gsi.x` and `enkf.x` builds to intel/2022 on NOAA RDHPCS machines.

Resolves #1863
---
 Externals.cfg       |  2 +-
 jobs/rocoto/anal.sh | 40 +++++++++++++++++++++++++++++++++++++---
 jobs/rocoto/eobs.sh | 40 +++++++++++++++++++++++++++++++++++++---
 jobs/rocoto/eupd.sh | 40 +++++++++++++++++++++++++++++++++++++---
 sorc/checkout.sh    |  2 +-
 5 files changed, 113 insertions(+), 11 deletions(-)

diff --git a/Externals.cfg b/Externals.cfg
index 325be174d2..eaf397edb9 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -36,7 +36,7 @@ protocol = git
 required = True
 
 [GSI-EnKF]
-hash = accb07e
+hash = ca19008
 local_path = sorc/gsi_enkf.fd
 repo_url = https://github.com/NOAA-EMC/GSI.git
 protocol = git
diff --git a/jobs/rocoto/anal.sh b/jobs/rocoto/anal.sh
index d99152ef19..cb07749b9e 100755
--- a/jobs/rocoto/anal.sh
+++ b/jobs/rocoto/anal.sh
@@ -4,9 +4,43 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-status=$?
-[[ ${status} -ne 0 ]] && exit ${status}
+#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#status=$?
+#[[ ${status} -ne 0 ]] && exit ${status}
+
+# TODO: clean this up
+#Source appropriate modulefiles based on machine
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
+    # Source FV3GFS workflow modules
+    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
+    status=$?
+    [[ ${status} -ne 0 ]] && exit "${status}"
+else
+    # Append compiler (only on machines that have multiple compilers)
+    COMPILER=${COMPILER:-"intel"}
+    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
+        MACHINE_ID=${MACHINE_ID}.${COMPILER}
+    fi
+
+    # Source machine specific GSI-EnKF modules
+    set +x
+    source "${HOMEgfs}/ush/module-setup.sh"
+    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
+    module load gsi_"${MACHINE_ID}"
+
+    if [[ "${MACHINE_ID}" = "orion" ]]; then
+       module load miniconda3/4.12.0
+       set +u
+       conda activate regional_workflow_cmaq
+       set_strict
+    fi
+
+    module list
+    unset MACHINE_ID
+    set_trace
+fi
 
 export job="anal"
 export jobid="${job}.$$"
diff --git a/jobs/rocoto/eobs.sh b/jobs/rocoto/eobs.sh
index 95fa42cb08..4f813a8d84 100755
--- a/jobs/rocoto/eobs.sh
+++ b/jobs/rocoto/eobs.sh
@@ -4,9 +4,43 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-status=$?
-[[ ${status} -ne 0 ]] && exit ${status}
+#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#status=$?
+#[[ ${status} -ne 0 ]] && exit ${status}
+
+# TODO: clean this up
+#Source appropriate modulefiles based on machine
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
+    # Source FV3GFS workflow modules
+    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
+    status=$?
+    [[ ${status} -ne 0 ]] && exit "${status}"
+else
+    # Append compiler (only on machines that have multiple compilers)
+    COMPILER=${COMPILER:-"intel"}
+    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
+        MACHINE_ID=${MACHINE_ID}.${COMPILER}
+    fi
+
+    # Source machine specific GSI-EnKF modules
+    set +x
+    source "${HOMEgfs}/ush/module-setup.sh"
+    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
+    module load gsi_"${MACHINE_ID}"
+
+    if [[ "${MACHINE_ID}" = "orion" ]]; then
+       module load miniconda3/4.12.0
+       set +u
+       conda activate regional_workflow_cmaq
+       set_strict
+    fi
+
+    module list
+    unset MACHINE_ID
+    set_trace
+fi
 
 export job="eobs"
 export jobid="${job}.$$"
diff --git a/jobs/rocoto/eupd.sh b/jobs/rocoto/eupd.sh
index 3ed028f87a..9ddde92d36 100755
--- a/jobs/rocoto/eupd.sh
+++ b/jobs/rocoto/eupd.sh
@@ -4,9 +4,43 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-status=$?
-[[ ${status} -ne 0 ]] && exit ${status}
+#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#status=$?
+#[[ ${status} -ne 0 ]] && exit ${status}
+
+# TODO: clean this up
+#Source appropriate modulefiles based on machine
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
+    # Source FV3GFS workflow modules
+    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
+    status=$?
+    [[ ${status} -ne 0 ]] && exit "${status}"
+else
+    # Append compiler (only on machines that have multiple compilers)
+    COMPILER=${COMPILER:-"intel"}
+    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
+        MACHINE_ID=${MACHINE_ID}.${COMPILER}
+    fi
+
+    # Source machine specific GSI-EnKF modules
+    set +x
+    source "${HOMEgfs}/ush/module-setup.sh"
+    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
+    module load gsi_"${MACHINE_ID}"
+
+    if [[ "${MACHINE_ID}" = "orion" ]]; then
+       module load miniconda3/4.12.0
+       set +u
+       conda activate regional_workflow_cmaq
+       set_strict
+    fi
+
+    module list
+    unset MACHINE_ID
+    set_trace
+fi
 
 export job="eupd"
 export jobid="${job}.$$"
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index c62d5b026d..9c9addad1d 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -156,7 +156,7 @@ checkout "ufs_model.fd"    "https://github.com/ufs-community/ufs-weather-model"
 checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git"   "c267780"                    ; errs=$((errs + $?))
 
 if [[ ${checkout_gsi} == "YES" ]]; then
-  checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "accb07e" "NO"; errs=$((errs + $?))
+  checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "ca19008" "NO"; errs=$((errs + $?))
 fi
 
 if [[ ${checkout_gdas} == "YES" ]]; then