Merge branch 'develop' into hotfix/segs

ci/Jenkinsfile

@@ -5,7 +5,7 @@ def cases = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
 def repo_url = '[email protected]:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 
@@ -101,7 +101,7 @@ pipeline {
                 stages {
                     stage('Building') {
                         steps {
-                           catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                          catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                             script {
                                 def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
                                 env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
@@ -173,9 +173,10 @@ pipeline {
                                     }
                                     if (system == 'gfs') {
                                         cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                        echo "Cases to run: ${cases}"
                                     }
                                }
-                           }
+                            }
                           }
                         }
                     }
@@ -193,22 +194,20 @@ pipeline {
                     def parallelStages = cases.collectEntries { caseName ->
                         ["${caseName}": {
                             stage("Create ${caseName}") {
-                                    catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                                        script {
-                                                def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
-                                                sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
-                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                                system = yaml_case.experiment.system
-                                                env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
-                                                try {
-                                                    error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
-                                                } catch (Exception error_create) {
-                                                    sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
-                                                    error("Case ${caseName} failed to create experment directory")
-                                                }
+                                script {
+                                        sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
+                                        def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                        def build_system = yaml_case.experiment.system
+                                        def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}"   // local HOMEgfs is used to populate the XML on per system basis
+                                        env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
+                                        env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
+                                        try {
+                                              error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
+                                        } catch (Exception error_create) {
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
+                                            error("Case ${caseName} failed to create experment directory")
                                         }
-                                }
+                                     }
                             }
 
                             stage("Running ${caseName}") {
@@ -219,8 +218,10 @@ pipeline {
                                                 def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
+                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                                def build_system = yaml_case.experiment.system
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
@@ -271,7 +272,6 @@ pipeline {
             }
         }
 
-
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
             steps {

ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -18,6 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - gaea
   - hercules

ush/python/pygfs/task/marine_analysis.py

@@ -210,6 +210,7 @@ def _prep_variational_yaml(self: Task) -> None:
         envconfig_jcb['cyc'] = os.getenv('cyc')
         envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER
         envconfig_jcb['obs_list'] = ['adt_rads_all']
+        envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES))
 
         # Write obs_list_short
         save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml')

ush/python/pygfs/utils/marine_da_utils.py

@@ -166,3 +166,36 @@ def clean_empty_obsspaces(config, target, app='var'):
 
     # save cleaned yaml
     save_as_yaml(config, target)
+
+
+@logit(logger)
+def get_mom6_levels(ocnres: str) -> int:
+    """
+    Temporary function that returns the number of vertical levels in MOM6 given the horizontal resolution.
+    This is requiered by the diffusion saber block that now makes use of oops::util::FieldSetHelpers::writeFieldSet
+    and requires the number of levels in the configuration. I have been told this will be changed in the future.
+
+    Parameters
+    -----------
+    ocnres: str
+        Input resolution for ocean in str format. e.g. '500', '100', '050', '025'
+
+   Returns
+   -------
+   nlev: int
+       number of levels in the ocean model given an input resolution
+    """
+
+    # Currently implemented resolutions
+    ocnres_to_nlev = {
+        '500': 25,
+        '100': 75,
+        '050': 75,
+        '025': 75
+    }
+    try:
+        nlev = ocnres_to_nlev.get(ocnres)
+    except KeyError:
+        raise KeyError("FATAL ERROR: Invalid ocnres value. Aborting.")
+
+    return nlev

versions/fix.ver

@@ -8,7 +8,7 @@ export cice_ver=20240416
 export cpl_ver=20230526
 export datm_ver=20220805
 export gdas_crtm_ver=20220805
-export gdas_fv3jedi_ver=20220805
+export gdas_fv3jedi_ver=20241022
 export gdas_soca_ver=20240802
 export gdas_gsibec_ver=20240416
 export gdas_obs_ver=20240213

workflow/generate_workflows.sh

@@ -71,6 +71,9 @@ function _usage() {
        If this option is not specified, then the existing email address in
        the crontab will be preserved.
 
+    -t Add a 'tag' to the end of the case names in the pslots to distinguish
+       pslots between multiple sets of tests.
+
     -v Verbose mode.  Prints output of all commands to stdout.
 
     -V Very verbose mode.  Passes -v to all commands and prints to stdout.
@@ -101,6 +104,7 @@ _hpc_account=""
 _set_account=false
 _update_cron=false
 _email=""
+_tag=""
 _set_email=false
 _verbose=false
 _very_verbose=false
@@ -111,7 +115,7 @@ _runtests="${RUNTESTS:-${_runtests:-}}"
 _nonflag_option_count=0
 
 while [[ $# -gt 0 && "$1" != "--" ]]; do
-   while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do
+   while getopts ":H:bB:uy:Y:GESA:ce:t:vVdh" option; do
       case "${option}" in
         H)
            HOMEgfs="${OPTARG}"
@@ -138,6 +142,7 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do
         S) _run_all_sfs=true ;;
         c) _update_cron=true ;;
         e) _email="${OPTARG}" && _set_email=true ;;
+        t) _tag="_${OPTARG}" ;;
         v) _verbose=true ;;
         V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;;
         d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;;
@@ -454,11 +459,12 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases"
 [[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}"
 for _case in "${_yaml_list[@]}"; do
    [[ "${_verbose}" == false ]] && echo "${_case}"
+   _pslot="${_case}${_tag}"
    _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite"
    if [[ "${_verbose}" == true ]]; then
-      pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd}
+      pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd}
    else
-      if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then
+      if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then
          _output=$(cat stdout stderr)
          _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status.  Output:"
          _message="${_message}"$'\n'"${_output}"
@@ -471,7 +477,7 @@ for _case in "${_yaml_list[@]}"; do
       fi
       rm -f stdout stderr
    fi
-   grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron
+   grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron
 done
 echo