Introduce version file support into develop branch (#1644)

Introduces the use of version files in the `develop` branch. This initial introduction
will use version files for runtime only. The ability to force version files for build is included
but commented out for now. A follow-up PR will turn on build version forcing when ready.

Changes include:

1. Update `module_base.*.lua` modulefiles to use module version variables and remove
hardcoded versions.
2. Introduce `build.<platform>.ver`, `run.<platform>.ver` files in the `/versions` folder.
3. Update `sorc/link_workflow.sh` to create `build.ver` and `run.ver` under `/versions` folder
based on `build.<platform>.ver` and `run.<platform>.ver`.
4. Introduce sourcing of `run.ver` in `ush/load_fv3gfs_modules.sh`
5. Introduce sourcing of `build.ver` in `sorc/build_all.sh` but commented out for now.
Components will continue setting their module versions for the time being.
6. Update `HOMEens_tracker` in `config.vrfy` to use `tracker_ver` from version files.
7. Replace obsproc version in `HOMEobsproc` setting in
`parm/config/gfs/config.base.emc.dyn` to use `obsproc_run_ver` from version files.

Resolves #671
Resolves #1625

.gitignore

@@ -143,3 +143,6 @@ ush/make_tif.sh
 ush/month_name.sh
 ush/imsfv3_scf2ioda.py
 ush/letkf_create_ens.py
+# version files
+versions/build.ver
+versions/run.ver

modulefiles/module_base.hera.lua

@@ -4,35 +4,36 @@ Load environment to run GFS on Hera
 
 prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
 
-load(pathJoin("hpc", "1.1.0"))
-load(pathJoin("hpc-intel", "18.0.5.274"))
-load(pathJoin("hpc-impi", "2018.0.4"))
-
-load(pathJoin("hpss", "hpss"))
-load(pathJoin("gempak", "7.4.2"))
-load(pathJoin("ncl", "6.6.2"))
-load(pathJoin("jasper", "2.0.25"))
-load(pathJoin("png", "1.6.35"))
-load(pathJoin("cdo", "1.9.5"))
-load(pathJoin("R", "3.5.0"))
-
-load(pathJoin("hdf5", "1.10.6"))
-load(pathJoin("netcdf", "4.7.4"))
-
-load(pathJoin("nco", "4.9.1"))
-load(pathJoin("prod_util", "1.2.2"))
-load(pathJoin("grib_util", "1.2.2"))
-load(pathJoin("g2tmpl", "1.10.0"))
-load(pathJoin("ncdiag", "1.0.0"))
-load(pathJoin("crtm", "2.4.0"))
-load(pathJoin("wgrib2", "2.0.8"))
+load(pathJoin("hpc", os.getenv("hpc_ver")))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
+load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+
+load(pathJoin("hpss", os.getenv("hpss_ver")))
+load(pathJoin("gempak", os.getenv("gempak_ver")))
+load(pathJoin("ncl", os.getenv("ncl_ver")))
+load(pathJoin("jasper", os.getenv("jasper_ver")))
+load(pathJoin("png", os.getenv("libpng_ver")))
+load(pathJoin("cdo", os.getenv("cdo_ver")))
+load(pathJoin("R", os.getenv("R_ver")))
+
+load(pathJoin("hdf5", os.getenv("hdf5_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+
+load(pathJoin("nco", os.getenv("nco_ver")))
+load(pathJoin("prod_util", os.getenv("prod_util_ver")))
+load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
+load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
 setenv("WGRIB2","wgrib2")
 
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
-load(pathJoin("prepobs", "1.0.1"))
+load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v1.0.0/modulefiles"))
-load(pathJoin("fit2obs", "1.0.0"))
+prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 -- Temporary until official hpc-stack is updated
 prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack")

modulefiles/module_base.jet.lua

@@ -4,40 +4,37 @@ Load environment to run GFS on Jet
 
 prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack")
 
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-intel", "18.0.5.274"))
-load(pathJoin("hpc-impi", "2018.4.274"))
-load(pathJoin("cmake", "3.20.1"))
+load(pathJoin("hpc", os.getenv("hpc_ver")))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
+load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
 
 load("hpss")
-load(pathJoin("gempak", "7.4.2"))
-load(pathJoin("ncl", "6.6.2"))
-load(pathJoin("jasper", "2.0.25"))
-load(pathJoin("libpng", "1.6.35"))
-load(pathJoin("cdo", "1.9.5"))
-load(pathJoin("R", "4.0.2"))
-
-load(pathJoin("hdf5", "1.10.6"))
-load(pathJoin("netcdf", "4.7.4"))
-
-load(pathJoin("nco", "4.9.1"))
-load(pathJoin("prod_util", "1.2.2"))
-load(pathJoin("grib_util", "1.2.2"))
-load(pathJoin("g2tmpl", "1.10.0"))
-load(pathJoin("ncdiag", "1.0.0"))
-load(pathJoin("crtm", "2.4.0"))
-load(pathJoin("wgrib2", "2.0.8"))
-setenv("WGRIB2","wgrib2")
-
-prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v1.0.1/modulefiles"))
-load(pathJoin("prepobs", "1.0.1"))
+load(pathJoin("gempak", os.getenv("gempak_ver")))
+load(pathJoin("ncl", os.getenv("ncl_ver")))
+load(pathJoin("jasper", os.getenv("jasper_ver")))
+load(pathJoin("libpng", os.getenv("libpng_ver")))
+load(pathJoin("cdo", os.getenv("cdo_ver")))
+load(pathJoin("R", os.getenv("R_ver")))
+
+load(pathJoin("hdf5", os.getenv("hdf5_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+
+load(pathJoin("nco", os.getenv("nco_ver")))
+load(pathJoin("prod_util", os.getenv("prod_util_ver")))
+load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
+load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
 
 prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-load(pathJoin("anaconda", "5.3.1"))
+load(pathJoin("anaconda", os.getenv("anaconda_ver")))
 
+--prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
-load(pathJoin("prepobs", "1.0.1"))
-prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v1.0.0/modulefiles"))
-load(pathJoin("fit2obs", "1.0.0"))
+load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
+
+prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 whatis("Description: GFS run environment")

modulefiles/module_base.orion.lua

@@ -4,34 +4,35 @@ Load environment to run GFS on Orion
 
 prepend_path("MODULEPATH", "/apps/contrib/NCEP/hpc-stack/libs/hpc-stack/modulefiles/stack")
 
-load(pathJoin("hpc", "1.1.0"))
-load(pathJoin("hpc-intel", "2018.4"))
-load(pathJoin("hpc-impi", "2018.4"))
-
-load(pathJoin("gempak", "7.5.1"))
-load(pathJoin("ncl", "6.6.2"))
-load(pathJoin("jasper", "2.0.25"))
-load(pathJoin("zlib", "1.2.11"))
-load(pathJoin("png", "1.6.35"))
-load(pathJoin("cdo", "1.9.5"))
-
-load(pathJoin("hdf5", "1.10.6"))
-load(pathJoin("netcdf", "4.7.4"))
-
-load(pathJoin("nco", "4.8.1"))
-load(pathJoin("prod_util", "1.2.2"))
-load(pathJoin("grib_util", "1.2.2"))
-load(pathJoin("g2tmpl", "1.10.0"))
-load(pathJoin("ncdiag", "1.0.0"))
-load(pathJoin("crtm", "2.4.0"))
-load(pathJoin("wgrib2", "2.0.8"))
+load(pathJoin("hpc", os.getenv("hpc_ver")))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
+load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+
+load(pathJoin("gempak", os.getenv("gempak_ver")))
+load(pathJoin("ncl", os.getenv("ncl_ver")))
+load(pathJoin("jasper", os.getenv("jasper_ver")))
+load(pathJoin("zlib", os.getenv("zlib_ver")))
+load(pathJoin("png", os.getenv("libpng_ver")))
+load(pathJoin("cdo", os.getenv("cdo_ver")))
+
+load(pathJoin("hdf5", os.getenv("hdf5_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+
+load(pathJoin("nco", os.getenv("nco_ver")))
+load(pathJoin("prod_util", os.getenv("prod_util_ver")))
+load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
+load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
 setenv("WGRIB2","wgrib2")
 
+--prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
-load(pathJoin("prepobs", "1.0.1"))
+load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v1.0.0/modulefiles"))
-load(pathJoin("fit2obs", "1.0.0"))
+prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 -- Temporary until official hpc-stack is updated
 prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack")

modulefiles/module_base.s4.lua

@@ -5,33 +5,34 @@ Load environment to run GFS on S4
 load("license_intel")
 prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack")
 
-load(pathJoin("hpc", "1.1.0"))
-load(pathJoin("hpc-intel", "18.0.4"))
-load(pathJoin("hpc-impi", "18.0.4"))
-
-load(pathJoin("miniconda", "3.8-s4"))
-load(pathJoin("ncl", "6.4.0-precompiled"))
-load(pathJoin("cdo", "1.9.8"))
-load(pathJoin("jasper", "2.0.25"))
-load(pathJoin("zlib", "1.2.11"))
-load(pathJoin("png", "1.6.35"))
-
-load(pathJoin("hdf5", "1.10.6"))
-load(pathJoin("netcdf", "4.7.4"))
-
-load(pathJoin("nco", "4.9.3"))
-load(pathJoin("prod_util", "1.2.2"))
-load(pathJoin("grib_util", "1.2.2"))
-load(pathJoin("g2tmpl", "1.10.0"))
-load(pathJoin("ncdiag", "1.0.0"))
-load(pathJoin("crtm", "2.4.0"))
-load(pathJoin("wgrib2", "2.0.8"))
+load(pathJoin("hpc", os.getenv("hpc_ver")))
+load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
+load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+
+load(pathJoin("miniconda", os.getenv("miniconda_ver")))
+load(pathJoin("ncl", os.getenv("ncl_ver")))
+load(pathJoin("cdo", os.getenv("cdo_ver")))
+load(pathJoin("jasper", os.getenv("jasper_ver")))
+load(pathJoin("zlib", os.getenv("zlib_ver")))
+load(pathJoin("png", os.getenv("libpng_ver")))
+
+load(pathJoin("hdf5", os.getenv("hdf5_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+
+load(pathJoin("nco", os.getenv("nco_ver")))
+load(pathJoin("prod_util", os.getenv("prod_util_ver")))
+load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
+load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
 setenv("WGRIB2","wgrib2")
 
+--prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg/modulefiles"))
-load(pathJoin("prepobs", "1.0.1"))
+load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v1.0.0/modulefiles"))
-load(pathJoin("fit2obs", "1.0.0"))
+prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 whatis("Description: GFS run environment")

modulefiles/module_base.wcoss2.lua

@@ -2,39 +2,40 @@ help([[
 Load environment to run GFS on WCOSS2
 ]])
 
-load(pathJoin("PrgEnv-intel", "8.1.0"))
-load(pathJoin("craype", "2.7.13"))
-load(pathJoin("intel", "19.1.3.304"))
-load(pathJoin("cray-mpich", "8.1.9"))
-load(pathJoin("cray-pals", "1.0.17"))
-load(pathJoin("cfp", "2.0.4"))
+load(pathJoin("PrgEnv-intel", os.getenv("PrgEnv_intel_ver")))
+load(pathJoin("craype", os.getenv("craype_ver")))
+load(pathJoin("intel", os.getenv("intel_ver")))
+load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver")))
+load(pathJoin("cray-pals", os.getenv("cray_pals_ver")))
+load(pathJoin("cfp", os.getenv("cfp_ver")))
 setenv("USE_CFP","YES")
 
-load(pathJoin("python", "3.8.6"))
-load(pathJoin("gempak", "7.14.1"))
-load(pathJoin("perl", "5.32.0"))
-load(pathJoin("libjpeg", "9c"))
-load(pathJoin("udunits", "2.2.28"))
-load(pathJoin("gsl", "2.7"))
-load(pathJoin("cdo", "1.9.8"))
+load(pathJoin("python", os.getenv("python_ver")))
+load(pathJoin("gempak", os.getenv("gempak_ver")))
+load(pathJoin("perl", os.getenv("perl_ver")))
+load(pathJoin("libjpeg", os.getenv("libjpeg_ver")))
+load(pathJoin("udunits", os.getenv("udunits_ver")))
+load(pathJoin("gsl", os.getenv("gsl_ver")))
+load(pathJoin("cdo", os.getenv("cdo_ver")))
 
-load(pathJoin("hdf5", "1.10.6"))
-load(pathJoin("netcdf", "4.7.4"))
+load(pathJoin("hdf5", os.getenv("hdf5_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
 
-load(pathJoin("nco", "4.7.9"))
-load(pathJoin("prod_util", "2.0.9"))
-load(pathJoin("grib_util", "1.2.3"))
-load(pathJoin("bufr_dump", "1.0.0"))
-load(pathJoin("util_shared", "1.4.0"))
-load(pathJoin("g2tmpl", "1.9.1"))
-load(pathJoin("ncdiag", "1.0.0"))
-load(pathJoin("crtm", "2.4.0"))
-load(pathJoin("wgrib2", "2.0.7"))
+load(pathJoin("nco", os.getenv("nco_ver")))
+load(pathJoin("prod_util", os.getenv("prod_util_ver")))
+load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("bufr_dump", os.getenv("bufr_dump_ver")))
+load(pathJoin("util_shared", os.getenv("util_shared_ver")))
+load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
+load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
 
+--prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
-load(pathJoin("prepobs", "1.0.1"))
+load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/Fit2Obs/v1.0.0/modulefiles"))
-load(pathJoin("fit2obs", "1.0.0"))
+prepend_path("MODULEPATH", pathJoin("/lfs/h2/emc/global/save/emc.global/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 whatis("Description: GFS run environment")

parm/config/gfs/config.base.emc.dyn

@@ -75,7 +75,7 @@ export MODE="@MODE@" # cycled/forecast-only
 # Build paths relative to $HOMEgfs
 export FIXgsi="${HOMEgfs}/fix/gsi"
 export HOMEpost="${HOMEgfs}"
-export HOMEobsproc="${BASE_GIT}/obsproc/v1.1.2"
+export HOMEobsproc="${BASE_GIT}/obsproc/v${obsproc_run_ver:-1.1.2}"
 
 # CONVENIENT utility scripts and other environment parameters
 export NCP="/bin/cp -p"

parm/config/gfs/config.vrfy

@@ -66,8 +66,7 @@ fi
 # Cyclone genesis and cyclone track verification
 #-------------------------------------------------
 
-export ens_tracker_ver=feature-GFSv17_com_reorg # TODO - temporary ahead of new tag/version
-export HOMEens_tracker=$BASE_GIT/TC_tracker/${ens_tracker_ver}
+export HOMEens_tracker=$BASE_GIT/TC_tracker/${tracker_ver}
 
 if [[ "${VRFYTRAK}" = "YES" ]]; then
 

sorc/build_all.sh

@@ -81,6 +81,12 @@ if [[ -z "${MACHINE_ID}" ]]; then
   exit 1
 fi
 
+#------------------------------------
+# SOURCE BUILD VERSION FILES
+#------------------------------------
+# TODO: Commented out until components aligned for build
+#source ../versions/build.ver
+
 #------------------------------------
 # INCLUDE PARTIAL BUILD
 #------------------------------------

sorc/link_workflow.sh

@@ -52,6 +52,12 @@ source gfs_utils.fd/ush/detect_machine.sh  # (sets MACHINE_ID)
 # shellcheck disable=
 machine=$(echo "${MACHINE_ID}" | cut -d. -f1)
 
+#------------------------------
+#--Set up build.ver and run.ver
+#------------------------------
+cp "${top_dir}/versions/build.${machine}.ver" "${top_dir}/versions/build.ver"
+cp "${top_dir}/versions/run.${machine}.ver" "${top_dir}/versions/run.ver"
+
 #------------------------------
 #--model fix fields
 #------------------------------

ush/load_fv3gfs_modules.sh

@@ -12,6 +12,9 @@ ulimit_s=$( ulimit -S -s )
 # Find module command and purge:
 source "${HOMEgfs}/modulefiles/module-setup.sh.inc"
 
+# Source versions file for runtime
+source "${HOMEgfs}/versions/run.ver"
+
 # Load our modules:
 module use "${HOMEgfs}/modulefiles"