Merge container into SFS_container

.gitmodules

@@ -1,28 +1,28 @@
 [submodule "sorc/ufs_model.fd"]
   path = sorc/ufs_model.fd
-  url = https://github.com/ufs-community/ufs-weather-model
+  url = https://github.com/benjamin-cash/ufs-weather-model
   ignore = dirty
 [submodule "sorc/wxflow"]
   path = sorc/wxflow
   url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
   path = sorc/gfs_utils.fd
-  url = https://github.com/NOAA-EMC/gfs-utils
+  url = https://github.com/benjamin-cash/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
   path = sorc/ufs_utils.fd
-  url = https://github.com/ufs-community/UFS_UTILS.git
+  url = https://github.com/benjamin-cash/UFS_UTILS.git
 [submodule "sorc/verif-global.fd"]
   path = sorc/verif-global.fd
   url = https://github.com/NOAA-EMC/EMC_verif-global.git
 [submodule "sorc/gsi_enkf.fd"]
   path = sorc/gsi_enkf.fd
-  url = https://github.com/NOAA-EMC/GSI.git
+  url = https://github.com/benjamin-cash/GSI.git
 [submodule "sorc/gdas.cd"]
   path = sorc/gdas.cd
   url = https://github.com/NOAA-EMC/GDASApp.git
 [submodule "sorc/gsi_utils.fd"]
   path = sorc/gsi_utils.fd
-  url = https://github.com/NOAA-EMC/GSI-Utils.git
+  url = https://github.com/benjamin-cash/GSI-Utils.git
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
-  url = https://github.com/NOAA-EMC/GSI-Monitor.git
+  url = https://github.com/benjamin-cash/GSI-Monitor.git

modulefiles/module_base.container.lua

@@ -0,0 +1,57 @@
+help([[
+Load environment to run GFS on Hera
+]])
+
+local spack_mod_path=(os.getenv("spack_mod_path") or "None")
+prepend_path("MODULEPATH", spack_mod_path)
+
+setenv("CC", "icc" ) 
+setenv("CXX","icpc" ) 
+setenv("F77","ifort" ) 
+setenv("F90","ifort" ) 
+setenv("FC", "ifort" ) 
+load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
+load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
+--load(pathJoin("python", (os.getenv("python_ver") or "None")))
+
+--load(pathJoin("hpss", (os.getenv("hpss_ver") or "None")))
+--load(pathJoin("gempak", (os.getenv("gempak_ver") or "None")))
+--load(pathJoin("ncl", (os.getenv("ncl_ver") or "None")))
+load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
+load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))
+load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
+--load(pathJoin("R", (os.getenv("R_ver") or "None")))
+--load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
+
+load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
+load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
+load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+
+load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
+load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
+load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None")))
+load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None")))
+load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None")))
+load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
+load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
+load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
+load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
+load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
+load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None")))
+load(pathJoin("met", (os.getenv("met_ver") or "None")))
+load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
+
+--setenv("WGRIB2","wgrib2")
+
+-- Stop gap fix for wgrib with spack-stack 1.6.0
+-- TODO Remove this when spack-stack issue #1097 is resolved
+--setenv("WGRIB","wgrib")
+--setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
+
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+
+whatis("Description: GFS run environment")

modulefiles/module_base.frontera.lua

@@ -0,0 +1,6 @@
+help([[
+Minimal environment to run GFS on Frontera via containers
+]])
+load("intel/23.1.0")  
+load("impi/21.9.0")  
+load("tacc-apptainer")

modulefiles/module_base.hopper.lua

@@ -0,0 +1,8 @@
+help([[
+Minimal environment to run GFS on Hopper via containers
+]])
+
+load("hosts/hopper")
+load("OneAPI/2022.1.2")
+load("singularity")
+load("mpi/2021.5.1")

run.container.ver

@@ -0,0 +1,36 @@
+export spack_stack_ver=1.6.0
+export python_ver=3.11.6
+
+export jasper_ver=2.0.32
+export libpng_ver=1.6.37
+export cdo_ver=2.3.0
+export nco_ver=5.1.6
+
+export hdf5_ver=1.14.3
+export netcdf_c_ver=4.9.2
+export netcdf_fortran_ver=4.6.1
+
+export bufr_ver=12.1.0
+export gsi_ncdiag_ver=1.1.2
+export g2tmpl_ver=1.13.0
+export crtm_ver=2.4.0.1
+export wgrib2_ver=3.1.1
+export grib_util_ver=1.4.0
+export prod_util_ver=2.1.1
+export py_netcdf4_ver=1.5.8
+export py_pyyaml_ver=6.0
+export py_jinja2_ver=3.1.2
+export py_pandas_ver=1.5.3
+export py_python_dateutil_ver=2.8.2
+export py_f90nml_ver=1.4.3
+export ip_ver=5.0.0
+
+export met_ver=11.1.1
+export metplus_ver=5.1.0
+export py_xarray_ver=2023.7.0
+
+export obsproc_run_ver=1.1.2
+export prepobs_run_ver=1.0.1
+
+export ens_tracker_ver=feature-GFSv17_com_reorg
+export fit2obs_ver=1.1.1

sorc/link_workflow.sh

@@ -77,6 +77,9 @@ case "${machine}" in
   "s4")       FIX_DIR="/data/prod/glopara/fix" ;;
   "gaea")     FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
   "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;;
+  "frontera") FIX_DIR="/work2/02441/bcash/frontera/ufs_input/fix" ;;
+  "container") FIX_DIR="/groups/BCASH/ufs_input/fix" ;;
+  "hopper") FIX_DIR="/groups/BCASH/ufs_input/fix" ;;
   *)
     echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR"
     exit 1

ush/detect_machine.sh

@@ -38,7 +38,8 @@ case $(hostname -f) in
   [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4
 
   login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4
-
+  *frontera.tacc.utexas.edu) MACHINE_ID=frontera ;;
+  *orc.gmu.edu) MACHINE_ID=hopper ;;
   login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2
 
   discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35
@@ -61,7 +62,10 @@ if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then
 fi
 
 # Try searching based on paths since hostname may not match on compute nodes
-if [[ -d /lfs/h3 ]]; then
+if [[ -d /opt/container-scripts ]]; then
+  # We are in the container
+  MACHINE_ID=container
+elif [[ -d /lfs/h3 ]]; then
   # We are on NOAA Cactus or Dogwood
   MACHINE_ID=wcoss2
 elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then

ush/load_fv3gfs_modules.sh

@@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver"
 module use "${HOMEgfs}/modulefiles"
 
 case "${MACHINE_ID}" in
-  "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
+  "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud" | "container" | "frontera" | "hopper")
     module load "module_base.${MACHINE_ID}"
     ;;
   *)

ush/module-setup.sh

@@ -3,7 +3,11 @@ set -u
 
 source "${HOMEgfs}/ush/detect_machine.sh"
 
-if [[ ${MACHINE_ID} = jet* ]] ; then
+if [[ ${MACHINE_ID} = container ]] ; then
+   source /usr/lmod/lmod/init/bash
+   module purge
+
+elif [[ ${MACHINE_ID} = jet* ]] ; then
     # We are on NOAA Jet
     if ( ! eval module help > /dev/null 2>&1 ) ; then
         source /apps/lmod/lmod/init/bash

versions/build.container.ver

@@ -0,0 +1,10 @@
+export stack_intel_ver=2021.10.0
+export stack_impi_ver=2021.12.1
+export spack_env=global-workflow-env
+source "${HOMEgfs:-}/versions/run.container.ver"
+export spack_mod_path="/opt/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"
+export I_MPI_CC=$CC
+export I_MPI_CXX=$CXX
+export I_MPI_F77=$FC
+export I_MPI_FC=$FC
+export I_MPI_F90=$FC

versions/run.container.ver

@@ -0,0 +1,43 @@
+export spack_stack_ver=1.6.0
+export python_ver=3.11.6
+
+export jasper_ver=2.0.32
+export libpng_ver=1.6.37
+export cdo_ver=2.3.0
+export nco_ver=5.1.6
+export esmf_ver=8.6.1
+export fms_ver=2024.02
+export mapl_ver=2.46.3-esmf-8.6.1
+export gftl_shared_ver=1.9.0
+
+export hdf5_ver=1.14.3
+export netcdf_c_ver=4.9.2
+export netcdf_fortran_ver=4.6.1
+
+export bufr_ver=12.1.0
+export gsi_ncdiag_ver=1.1.2
+export g2tmpl_ver=1.13.0
+export g2_ver=3.5.1 
+export cmake_ver=3.27.9
+export crtm_ver=2.4.0.1
+export wgrib2_ver=3.1.1
+export grib_util_ver=1.4.0
+export prod_util_ver=2.1.1
+export py_netcdf4_ver=1.5.8
+export py_pyyaml_ver=6.0
+export py_jinja2_ver=3.1.2
+export parallelio_ver=2.6.2
+export py_pandas_ver=1.5.3
+export py_python_dateutil_ver=2.8.2
+export py_f90nml_ver=1.4.3
+export ip_ver=5.0.0
+
+export met_ver=11.1.1
+export metplus_ver=5.1.0
+export py_xarray_ver=2023.7.0
+
+export obsproc_run_ver=1.1.2
+export prepobs_run_ver=1.0.1
+
+export ens_tracker_ver=feature-GFSv17_com_reorg
+export fit2obs_ver=1.1.1

versions/spack.ver

@@ -0,0 +1,45 @@
+export spack_stack_ver=1.8.0
+export python_ver=3.11.6
+export stack_intel_ver=2021.10.0
+export stack_impi_ver=2021.12.1
+
+export jasper_ver=2.0.32
+export libpng_ver=1.6.37
+export cdo_ver=2.3.0
+export nco_ver=5.1.6
+export esmf_ver=8.6.1
+export fms_ver=2024.02
+export mapl_ver=2.46.3-esmf-8.6.1
+export gftl_shared_ver=1.9.0
+
+export hdf5_ver=1.14.3
+export netcdf_c_ver=4.9.2
+export netcdf_fortran_ver=4.6.1
+
+export bufr_ver=12.1.0
+export gsi_ncdiag_ver=1.1.2
+export g2tmpl_ver=1.13.0
+export g2_ver=3.5.1
+export cmake_ver=3.27.9
+export crtm_ver=2.4.0.1
+export wgrib2_ver=3.1.1
+export grib_util_ver=1.4.0
+export prod_util_ver=2.1.1
+export py_netcdf4_ver=1.5.8
+export py_pyyaml_ver=6.0
+export py_jinja2_ver=3.1.2
+export parallelio_ver=2.6.2
+export py_pandas_ver=1.5.3
+export py_python_dateutil_ver=2.8.2
+export py_f90nml_ver=1.4.3
+export ip_ver=5.0.0
+
+export met_ver=11.1.1
+export metplus_ver=5.1.0
+export py_xarray_ver=2023.7.0
+
+export obsproc_run_ver=1.1.2
+export prepobs_run_ver=1.0.1
+
+export ens_tracker_ver=feature-GFSv17_com_reorg
+export fit2obs_ver=1.1.1

workflow/hosts.py

@@ -17,7 +17,8 @@ class Host:
 
     SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES',
                        'WCOSS2', 'S4', 'CONTAINER', 'GAEA',
-                       'AWSPW', 'AZUREPW', 'GOOGLEPW']
+                       'AWSPW', 'AZUREPW', 'GOOGLEPW',
+                       'HOPPER', 'FRONTERA']
 
     def __init__(self, host=None):