From 39d0b8b12ae3afd781e7c0bcb83e4573f1a4dfc7 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Fri, 25 Oct 2024 09:22:42 -0400 Subject: [PATCH 1/3] Add ability to add tag to pslots with generate_workflows (#3036) # Description `generate_workflows.sh` would only use the case name as the pslot, leading to conflicts if you try to run two different sets at the same time. Now there is a new option that allows a 'tag' to the case name when determining the pslots: ``` generate_workflows.sh -t tag: string to be added to end of case name ``` For example, this: ``` generate_workflows.sh -t test ``` Will result in pslots of `C48_ATM_test`, `C48_S2SW_test`, etc. This is similar to how the CI system appends the hash to the pslots, but allows flexibility since this is a command-line tool. # Type of change - [ ] Bug fix (fixes something broken) - [x] New feature (adds functionality) - [ ] Maintenance (code refactor, clean-up, new CI test, etc.) # Change characteristics - Is this a breaking change (a change in existing functionality)? NO - Does this change require a documentation update? NO - Does this change require an update to any of the following submodules? NO # How has this been tested? - Ran `generate_workflows.sh` both with and without the new `-t` option. # Checklist - [x] Any dependent changes have been merged and published - [x] My code follows the style guidelines of this project - [x] I have performed a self-review of my own code - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have documented my code, including function, input, and output descriptions - [x] My changes generate no new warnings - [x] New and existing tests pass with my changes - [ ] This change is covered by an existing CI test or a new one has been added - [x] Any new scripts have been added to the .github/CODEOWNERS file with owners - [x] I have made corresponding changes to the system documentation if necessary --- workflow/generate_workflows.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/workflow/generate_workflows.sh b/workflow/generate_workflows.sh index c4f89eef6e..c2565140d3 100755 --- a/workflow/generate_workflows.sh +++ b/workflow/generate_workflows.sh @@ -71,6 +71,9 @@ function _usage() { If this option is not specified, then the existing email address in the crontab will be preserved. + -t Add a 'tag' to the end of the case names in the pslots to distinguish + pslots between multiple sets of tests. + -v Verbose mode. Prints output of all commands to stdout. -V Very verbose mode. Passes -v to all commands and prints to stdout. @@ -101,6 +104,7 @@ _hpc_account="" _set_account=false _update_cron=false _email="" +_tag="" _set_email=false _verbose=false _very_verbose=false @@ -111,7 +115,7 @@ _runtests="${RUNTESTS:-${_runtests:-}}" _nonflag_option_count=0 while [[ $# -gt 0 && "$1" != "--" ]]; do - while getopts ":H:bB:uy:Y:GESA:ce:vVdh" option; do + while getopts ":H:bB:uy:Y:GESA:ce:t:vVdh" option; do case "${option}" in H) HOMEgfs="${OPTARG}" @@ -138,6 +142,7 @@ while [[ $# -gt 0 && "$1" != "--" ]]; do S) _run_all_sfs=true ;; c) _update_cron=true ;; e) _email="${OPTARG}" && _set_email=true ;; + t) _tag="_${OPTARG}" ;; v) _verbose=true ;; V) _very_verbose=true && _verbose=true && _verbose_flag="-v" ;; d) _debug=true && _very_verbose=true && _verbose=true && _verbose_flag="-v" && PS4='${LINENO}: ' ;; @@ -454,11 +459,12 @@ echo "Running create_experiment.py for ${#_yaml_list[@]} cases" [[ "${_verbose}" == true ]] && printf "Selected cases: %s\n\n" "${_yaml_list[*]}" for _case in "${_yaml_list[@]}"; do [[ "${_verbose}" == false ]] && echo "${_case}" + _pslot="${_case}${_tag}" _create_exp_cmd="./create_experiment.py -y ../ci/cases/pr/${_case}.yaml --overwrite" if [[ "${_verbose}" == true ]]; then - pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} + pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} else - if ! pslot=${_case} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then + if ! pslot=${_pslot} RUNTESTS=${_runtests} ${_create_exp_cmd} 2> stderr 1> stdout; then _output=$(cat stdout stderr) _message="The create_experiment command (${_create_exp_cmd}) failed with a non-zero status. Output:" _message="${_message}"$'\n'"${_output}" @@ -471,7 +477,7 @@ for _case in "${_yaml_list[@]}"; do fi rm -f stdout stderr fi - grep "${_case}" "${_runtests}/EXPDIR/${_case}/${_case}.crontab" >> tests.cron + grep "${_pslot}" "${_runtests}/EXPDIR/${_pslot}/${_pslot}.crontab" >> tests.cron done echo From 2fdfaa5a39a71544a907bdcc2dbd9559acf60a2b Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Sat, 26 Oct 2024 18:00:16 -0400 Subject: [PATCH 2/3] Update gdas.cd (#2978) Updates to the `gdas.cd` #. @RussTreadon-NOAA will submit a PR in the GDASApp, we'll update the gdas.cd # in this branch after the GDASApp PR is merged. In the mean time, could somebody review the few simple code changes that are needed to run with the new #? - [x] Depends on GDASApp PR [#1310](https://github.com/NOAA-EMC/GDASApp/pull/1310) - [x] Depends on g-w issue [#3012](https://github.com/NOAA-EMC/global-workflow/issues/3012) --------- Co-authored-by: Rahul Mahajan Co-authored-by: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Co-authored-by: RussTreadon-NOAA --- ci/Jenkinsfile | 2 +- ci/cases/pr/C96C48_hybatmaerosnowDA.yaml | 1 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/marine_analysis.py | 1 + ush/python/pygfs/utils/marine_da_utils.py | 33 +++++++++++++++++++++++ versions/fix.ver | 2 +- 6 files changed, 38 insertions(+), 3 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 1a63e104bb..639e0ff223 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -5,7 +5,7 @@ def cases = '' def GH = 'none' // Location of the custom workspaces for each machine in the CI system. They are persitent for each iteration of the PR. def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea'] -def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] +def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI'] def repo_url = 'git@github.com:NOAA-EMC/global-workflow.git' def STATUS = 'Passed' diff --git a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml index be5ad32238..57d0989ae3 100644 --- a/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml +++ b/ci/cases/pr/C96C48_hybatmaerosnowDA.yaml @@ -18,6 +18,7 @@ arguments: yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml skip_ci_on_hosts: + - wcoss2 - orion - gaea - hercules diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 55e895f1dc..764f58cebd 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 55e895f1dcf4e6be36eb0eb4c8a7995d429157e0 +Subproject commit 764f58cebdf64f3695d89538994a50183e5884d9 diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index 4e4311b906..e7b7b5e948 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -210,6 +210,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['cyc'] = os.getenv('cyc') envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER envconfig_jcb['obs_list'] = ['adt_rads_all'] + envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES)) # Write obs_list_short save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml') diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index e1b2ac2d4d..50d9d84e86 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -166,3 +166,36 @@ def clean_empty_obsspaces(config, target, app='var'): # save cleaned yaml save_as_yaml(config, target) + + +@logit(logger) +def get_mom6_levels(ocnres: str) -> int: + """ + Temporary function that returns the number of vertical levels in MOM6 given the horizontal resolution. + This is requiered by the diffusion saber block that now makes use of oops::util::FieldSetHelpers::writeFieldSet + and requires the number of levels in the configuration. I have been told this will be changed in the future. + + Parameters + ----------- + ocnres: str + Input resolution for ocean in str format. e.g. '500', '100', '050', '025' + + Returns + ------- + nlev: int + number of levels in the ocean model given an input resolution + """ + + # Currently implemented resolutions + ocnres_to_nlev = { + '500': 25, + '100': 75, + '050': 75, + '025': 75 + } + try: + nlev = ocnres_to_nlev.get(ocnres) + except KeyError: + raise KeyError("FATAL ERROR: Invalid ocnres value. Aborting.") + + return nlev diff --git a/versions/fix.ver b/versions/fix.ver index 7c18bea081..b175791196 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -8,7 +8,7 @@ export cice_ver=20240416 export cpl_ver=20230526 export datm_ver=20220805 export gdas_crtm_ver=20220805 -export gdas_fv3jedi_ver=20220805 +export gdas_fv3jedi_ver=20241022 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 From b56f633d4a0feaf65955e19f01d1e888b4120db4 Mon Sep 17 00:00:00 2001 From: TerrenceMcGuinness-NOAA Date: Sun, 27 Oct 2024 21:20:54 -0400 Subject: [PATCH 3/3] CI jekninsfile update hotfix (#3038) This PR is a needed hotfix to the Jenkinsfile pipeline script that fixes a bug with the matrix control variable name clash with the variable **system** for the two build types. --- ci/Jenkinsfile | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 639e0ff223..6a2e064be0 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -101,7 +101,7 @@ pipeline { stages { stage('Building') { steps { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { + catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { script { def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc @@ -173,9 +173,10 @@ pipeline { } if (system == 'gfs') { cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split() + echo "Cases to run: ${cases}" } } - } + } } } } @@ -193,22 +194,20 @@ pipeline { def parallelStages = cases.collectEntries { caseName -> ["${caseName}": { stage("Create ${caseName}") { - catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') { - script { - def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to populate the XML on per system basis - env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc - sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") - def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" - system = yaml_case.experiment.system - env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" - try { - error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() - } catch (Exception error_create) { - sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) - error("Case ${caseName} failed to create experment directory") - } + script { + sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system + def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}" // local HOMEgfs is used to populate the XML on per system basis + env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc + env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS" + try { + error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim() + } catch (Exception error_create) { + sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """) + error("Case ${caseName} failed to create experment directory") } - } + } } stage("Running ${caseName}") { @@ -219,8 +218,10 @@ pipeline { def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim() def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs" sh(script: " rm -f ${error_file}") + def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp" + def build_system = yaml_case.experiment.system try { - sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}") + sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}") } catch (Exception error_experment) { sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}") ws(CUSTOM_WORKSPACE) { @@ -271,7 +272,6 @@ pipeline { } } - stage( '5. FINALIZE' ) { agent { label NodeName[machine].toLowerCase() } steps {