Merge pull request NCAR#406 from mkavulich/feature/derecho

Add module files for building SCM with spack-stack on Derecho, Hera, Jet, Orion

scm/doc/TechGuide/chap_quick.tex

@@ -99,19 +99,15 @@ \subsection{Compilers}
 release website (\url{https://dtcenter.org/community-code/common-community-physics-package-ccpp/download}).
 
 \subsection{Using Existing Libraries on Preconfigured Platforms}\label{section: use_preconfigured_platforms}
-Platform-specific scripts are provided to load modules and set the user environment for preconfigured platforms. These scripts load compiler modules (Fortran 2008-compliant), the NetCDF module, Python environment, etc. and set compiler and environment variables. From the top-level code directory (\execout{ccpp-scm} by default), source the correct script for your platform and shell. For \textit{t/csh} shells,
-\begin{lstlisting}[language=csh]
-source scm/etc/Hera_setup_intel.csh
-source scm/etc/Cheyenne_setup_gnu.csh
-source scm/etc/Cheyenne_setup_intel.csh
-\end{lstlisting}
-For bourne/bash shells,
-\begin{lstlisting}[language=bash]
-. scm/etc/Hera_setup_intel.sh
-. scm/etc/Cheyenne_setup_gnu.sh
-. scm/etc/Cheyenne_setup_intel.sh
+Because the SCM can be built using the so-called \href{https://ufs-weather-model.readthedocs.io/en/latest/Glossary.html#term-spack-stack}{"spack-stack" libraries} maintained for the UFS Weather Model effort, there are many platforms where the SCM can be built using those existing libraries. This can be done by loading provided modules in the \execout{scm/etc/modules} directory (must be done from the top-level "ccpp-scm" directory; otherwise the \execout{module use} command should point to the corresponding absolute path):
+
+\begin{lstlisting}[language=sh]
+module purge
+module use scm/etc/modules
+module load [machine]_[compiler]
 \end{lstlisting}
 
+View the contents of the \execout{scm/etc/modules} directory to see if your machine/compiler combination is supported. As of this writing, modulefiles are provided for Intel and GNU compilers on the NCAR machine Derecho, the NOAA machines Hera and Jet, and the NOAA/MSU machine Orion. Loading these modules will set up all the needed compilers, libraries, and other programs needed for building, as well as the python libraries needed for both building and running the SCM.
 
 \subsection{Installing Libraries on Non-preconfigured Platforms}\label{section: setup_supported_platforms}
 For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, the \execout{hpc-stack} project is suggested for installing prerequisite libraries. Visit \url{https://github.com/NOAA-EMC/hpc-stack} for instructions for installing prerequisite libraries via \execout{hpc-stack} in their docs directory. UFS users who already installed libraries via the \execout{hpc-stack} package only need to set the compiler (\execout{CC, CXX, FC}), NetCDF (\execout{NetCDF\_ROOT}), and \execout{bacio},  \execout{sp} and \execout{w3emc} (\execout{bacio\_ROOT}, \execout{sp\_ROOT}, \execout{w3emc\_ROOT}) environment variables to point to their installation paths in order to compile the SCM. 

scm/doc/TechGuide/chap_repo.tex

@@ -26,32 +26,21 @@ \section{What is included in the repository?}
 	.3 Dockerfile\DTcomment{contains Docker instructions for building the CCPP SCM image}.
 	.2 README.md.
 	.2 scm/.
-	.3 bin/\DTcomment{build directory (initially empty; populated by cmake)}.
 	.3 data/\DTcomment{build directory (most data directories populated by contrib/get\_all\_static\_data.sh)}.
-	.4 comparison\_data/\DTcomment{initially empty; contains data with which to compare SCM output}.
-	.4 physics\_input\_data/\DTcomment{initially empty; contains data needed by the CCPP physics}.
-	.4 processed\_case\_input/\DTcomment{initially empty; contains initialization and forcing data for cases}.
-	.4 raw\_case\_input/\DTcomment{initially empty; contains case data to be processed by scripts}.
 	.4 vert\_coord\_data/\DTcomment{contains data to calculate vertical coordinates (from GSM-based GFS only)}.
 	.3 doc/\DTcomment{contains this User's/Technical Guide}.
 	.4 TechGuide/\DTcomment{contains LaTeX for this User's Guide}.
 	.3 etc/\DTcomment{contains case configuration, machine setup scripts, and plotting scripts}.
 	.4 case\_config/\DTcomment{contains case configuration files}.
 	.4 CENTOS\_docker\_setup.sh\DTcomment{contains machine setup for Docker container}.
-	.4 Cheyenne\_setup\_gnu.csh\DTcomment{setup script for Cheyenne HPC for csh, tcsh}.
-	.4 Cheyenne\_setup\_gnu.sh\DTcomment{setup script for Cheyenne HPC for sh, bash}.
-	.4 Cheyenne\_setup\_intel.csh\DTcomment{setup script for Cheyenne HPC for csh, tcsh}.
-	.4 Cheyenne\_setup\_intel.sh\DTcomment{setup script for Cheyenne HPC for sh, bash}.
 	.4 Desktop\_setup\_gfortran.csh\DTcomment{setup script for Mac Desktop for csh, tcsh}.
 	.4 Desktop\_setup\_gfortran.sh\DTcomment{setup script for Mac Desktop for sh, bash}.
-	.4 Hera\_setup\_intel.csh\DTcomment{setup script for Theia HPC for csh, tcsh}.
-	.4 Hera\_setup\_intel.sh\DTcomment{setup script for Theia HPC for sh, bash}.
+        .4 modules/\DTcomment{Directory containing modulefiles for building on HPC environments (see section \ref{section: use_preconfigured_platforms})}.
 	.4 scm\_qsub\_example.py\DTcomment{example QSUB run script}.
 	.4 scm\_slurm\_example.py\DTcomment{example SLURM run script}.
 	.4 scripts/\DTcomment{Python scripts for setting up cases and plotting}.
 	.5 plot\_configs/\DTcomment{plot configuration files}.
 	.4 tracer\_config\DTcomment{tracer configuration files}.
 	.3 LICENSE.txt.
-	.3 run/\DTcomment{initially empty; populated by run\_scm.py}.
 	.3 src/\DTcomment{source code for SCM infrastructure, Python run script, CMakeLists.txt for the SCM, example multirun setup files, suite\_info.py}.
 }}

scm/etc/modules/derecho_gnu.lua

@@ -0,0 +1,30 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the CISL machine Derecho (Cray) using Intel-classic-2023.0.0
+]])
+
+whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with GNU compilers]===])
+
+setenv("LMOD_TMOD_FIND_FIRST","yes")
+load("ncarenv/23.09")
+load("cmake/3.26.3")
+
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles")
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-gcc/12.2.0")
+load("stack-cray-mpich/8.1.25")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpicc")
+setenv("CMAKE_CXX_COMPILER","mpicxx")
+setenv("CMAKE_Fortran_COMPILER","mpif90")
+setenv("CMAKE_Platform","derecho.gnu")

scm/etc/modules/derecho_intel.lua

@@ -0,0 +1,30 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the CISL machine Derecho (Cray) using Intel-classic-2023.0.0
+]])
+
+whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with Intel compilers]===])
+
+setenv("LMOD_TMOD_FIND_FIRST","yes")
+load("ncarenv/23.09")
+load("cmake/3.26.3")
+
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles")
+prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-intel/2021.10.0")
+load("stack-cray-mpich/8.1.25")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","cc")
+setenv("CMAKE_CXX_COMPILER","CC")
+setenv("CMAKE_Fortran_COMPILER","ftn")
+setenv("CMAKE_Platform","derecho.intel")

scm/etc/modules/hera_gnu.lua

@@ -0,0 +1,30 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the NOAA RDHPC machine Hera using GNU 9.2.0
+]])
+
+whatis([===[Loads libraries needed for building the CCPP SCM on Hera with GNU compilers ]===])
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
+
+load("cmake/3.20.1")
+load("miniconda/3.9.12")
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-gcc/9.2.0")
+load("stack-openmpi/4.1.5")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpicc")
+setenv("CMAKE_CXX_COMPILER","mpicxx")
+setenv("CMAKE_Fortran_COMPILER","mpif90")
+setenv("CMAKE_Platform","hera.gnu")

scm/etc/modules/hera_intel.lua

@@ -0,0 +1,29 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the NOAA RDHPC machine Hera using Intel-2021.5.0
+]])
+
+whatis([===[Loads libraries needed for building the CCPP SCM on Hera with Intel compilers ]===])
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
+load("cmake/3.20.1")
+load("miniconda/3.9.12")
+
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-intel/2021.5.0")
+load("stack-intel-oneapi-mpi/2021.5.1")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpiicc")
+setenv("CMAKE_CXX_COMPILER","mpiicpc")
+setenv("CMAKE_Fortran_COMPILER","mpiifort")
+setenv("CMAKE_Platform","hera.intel")

scm/etc/modules/jet_gnu.lua

@@ -0,0 +1,31 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the NOAA RDHPC machine Jet using GNU 9.2.0
+]])
+
+whatis([===[Loads libraries needed for building the CCPP SCM on Jet with GNU compilers ]===])
+
+prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles")
+
+load("cmake/3.26.4")
+load("miniconda/3.9.12")
+
+prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/modulefiles")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-gcc/9.2.0")
+load("stack-openmpi/3.1.4")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpicc")
+setenv("CMAKE_CXX_COMPILER","mpicxx")
+setenv("CMAKE_Fortran_COMPILER","mpif90")
+setenv("CMAKE_Platform","jet.gnu")

scm/etc/modules/jet_intel.lua

@@ -0,0 +1,31 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the NOAA RDHPC machine Jet using Intel-2021.5.0
+]])
+
+whatis([===[Loads libraries needed for building the CCPP SCM on Jet with Intel compilers ]===])
+
+prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles")
+
+load("cmake/3.26.4")
+load("miniconda/3.9.12")
+
+prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/modulefiles")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-intel/2021.5.0")
+load("stack-intel-oneapi-mpi/2021.5.1")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpiicc")
+setenv("CMAKE_CXX_COMPILER","mpiicpc")
+setenv("CMAKE_Fortran_COMPILER","mpiifort")
+setenv("CMAKE_Platform","jet.intel")

scm/etc/modules/orion_gnu.lua

@@ -0,0 +1,30 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the NOAA RDHPC machine orion using GNU 10.2.0
+]])
+
+whatis([===[Loads libraries needed for building the CCPP SCM on Orion with GNU compilers ]===])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/modulefiles")
+
+load("cmake/3.22.1")
+load("python/3.9.2")
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-gcc/10.2.0")
+load("stack-openmpi/4.0.4")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpicc")
+setenv("CMAKE_CXX_COMPILER","mpicxx")
+setenv("CMAKE_Fortran_COMPILER","mpif90")
+setenv("CMAKE_Platform","orion.gnu")

scm/etc/modules/orion_intel.lua

@@ -0,0 +1,30 @@
+help([[
+This module loads libraries for building the CCPP Single-Column Model on
+the NOAA RDHPC machine Orion using Intel-2021.5.0
+]])
+
+whatis([===[Loads libraries needed for building the CCPP SCM on Orion with Intel compilers ]===])
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/modulefiles")
+
+load("cmake/3.22.1")
+load("python/3.9.2")
+
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core")
+
+load("stack-intel/2022.0.2")
+load("stack-intel-oneapi-mpi/2021.5.1")
+load("stack-python/3.10.8")
+load("py-f90nml")
+load("py-netcdf4/1.5.8")
+
+load("netcdf-c/4.9.2")
+load("netcdf-fortran/4.6.0")
+load("bacio/2.4.1")
+load("sp/2.3.3")
+load("w3emc")
+
+setenv("CMAKE_C_COMPILER","mpiicc")
+setenv("CMAKE_CXX_COMPILER","mpiicpc")
+setenv("CMAKE_Fortran_COMPILER","mpiifort")
+setenv("CMAKE_Platform","orion.intel")

scm/src/CMakeLists.txt

@@ -30,6 +30,9 @@ endif()
 message (STATUS "Running ccpp_prebuild.py for CCPP")
 # Make the directories where the ccpp_prebuild.py script wants to write caps and make snippets
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/physics)
+if (NOT EXISTS "${CMAKE_SOURCE_DIR}/../../ccpp/framework/scripts/ccpp_prebuild.py")
+  message( FATAL_ERROR "ccpp_prebuild.py script does not exist, did you check out the code recursively?" )
+endif()
 if (${CMAKE_BUILD_TYPE} MATCHES "Debug")
   execute_process(
     COMMAND ccpp/framework/scripts/ccpp_prebuild.py --config=ccpp/config/ccpp_prebuild_config.py ${_ccpp_suites_arg} --builddir=${CMAKE_CURRENT_BINARY_DIR} --debug --verbose

scm/src/run_scm.py

@@ -758,13 +758,16 @@ def main():
      verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out,       \
      n_itt_diag, run_dir, bin_dir, timestep) = parse_arguments()
 
+    setup_logging(verbose)
+
     global SCM_ROOT
     SCM_ROOT = os.getenv('SCM_ROOT')
     if SCM_ROOT is None:
-        message = 'The SCM_ROOT environment variable is not set. Please set the SCM_ROOT environment variable to the top-level path where the model was cloned.'
-        logging.critical(message)
-        raise Exception(message)
-
+        # Note: os.path.dirname() is the platform-agnostic way to get a parent directory
+        SCM_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+        message = f"The SCM_ROOT environment variable is not set. Using default value:\n{SCM_ROOT}"
+        logging.warning(message)
+
     global SCM_BIN
     if bin_dir:
         SCM_BIN = bin_dir
@@ -782,14 +785,11 @@ def main():
     global EXECUTABLE
     EXECUTABLE = os.path.join(SCM_RUN, EXECUTABLE_NAME)
 
-    setup_logging(verbose)
-
     # Debugger
     if use_gdb:
         gdb = find_gdb()
     else:
         gdb = None
-
     if (file != None):
         logging.info('SCM-run: Using {} to loop through defined runs'.format(file))
         try: