Merge branch 'develop' into feature/jedi_class

.github/pull_request_template.md

@@ -33,6 +33,17 @@
 # Change characteristics
 - Is this a breaking change (a change in existing functionality)? YES/NO
 - Does this change require a documentation update? YES/NO
+- Does this change require an update to any of the following submodules? YES/NO (If YES, please add a link to any PRs that are pending.)
+  - [ ] EMC verif-global
+  - [ ] GDAS
+  - [ ] GFS-utils
+  - [ ] GSI
+  - [ ] GSI-monitor
+  - [ ] GSI-utils
+  - [ ] UFS-utils
+  - [ ] UFS-weather-model
+  - [ ] wxflow
+
 
 # How has this been tested?
 <!-- Please list any test you conducted, including the machine.

ci/Jenkinsfile

@@ -282,10 +282,10 @@ pipeline {
             steps {
                 script {
                     sh(script: """
-                        labels=\$(gh pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
+                        labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
                         for label in \$labels; do
                             if [[ "\$label" == *"${Machine}"* ]]; then
-                                gh pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "\$label"
+                                ${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --remove-label "\$label"
                             fi
                         done
                     """, returnStatus: true)

ci/cases/gfsv17/ocnanal.yaml

@@ -17,7 +17,7 @@ base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 
 ocnanal:
-  SOCA_INPUT_FIX_DIR: {{ FIXgfs }}/gdas/soca/1440x1080x75/soca
+  SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
 

ci/scripts/check_ci.sh

@@ -50,14 +50,14 @@ fi
 export GH
 
 rocotostat=$(command -v rocotostat)
-if [[ -z ${rocotostat+x} ]]; then
+if [[ -z ${rocotostat} ]]; then
   echo "rocotostat not found on system"
   exit 1
 else
   echo "rocotostat being used from ${rocotostat}"
 fi
 rocotocheck=$(command -v rocotocheck)
-if [[ -z ${rocotocheck+x} ]]; then
+if [[ -z ${rocotocheck} ]]; then
   echo "rocotocheck not found on system"
   exit 1
 else
@@ -70,7 +70,7 @@ pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
   pr_list=$("${HOMEgfs}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Running) || true
 fi
-if [[ -z "${pr_list+x}" ]]; then
+if [[ -z "${pr_list}" ]]; then
   echo "no PRs open and ready to run cases on .. exiting"
   exit 0
 fi
@@ -124,7 +124,7 @@ for pr in ${pr_list}; do
 
   for pslot_dir in "${pr_dir}/RUNTESTS/EXPDIR/"*; do
     pslot=$(basename "${pslot_dir}") || true
-    if [[ -z "${pslot+x}" ]]; then
+    if [[ -z "${pslot}" ]]; then
       echo "No experiments found in ${pslot_dir} .. exiting"
       exit 0
     fi

ci/scripts/driver.sh

@@ -77,8 +77,9 @@ pr_list=$(${GH} pr list --repo "${REPO_URL}" --label "CI-${MACHINE_ID^}-Ready" -
 
 for pr in ${pr_list}; do
   pr_dir="${GFS_CI_ROOT}/PR/${pr}"
+  [[ ! -d ${pr_dir} ]] && mkdir -p "${pr_dir}"
   db_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --add_pr "${pr}" --dbfile "${pr_list_dbfile}")
-  output_ci_single="${GFS_CI_ROOT}/PR/${pr}/output_single.log"
+  output_ci_single="${pr_dir}/output_single.log"
   #############################################################
   # Check if a Ready labeled PR has changed back from once set
   # and in that case completely kill the previose driver.sh cron
@@ -107,7 +108,9 @@ for pr in ${pr_list}; do
            echo -e "${pstree_out}" | grep -Pow "(?<=\()[0-9]+(?=\))" | xargs kill
         fi
       else
-        ssh "${driver_HOST}" 'pstree -A -p "${driver_PID}" | grep -Eow "[0-9]+" | xargs kill'
+        # Check if the driver is still running on the head node; if so, kill it and all child processes
+        #shellcheck disable=SC2029
+        ssh "${driver_HOST}" "pstree -A -p \"${driver_PID}\" | grep -Eow \"[0-9]+\" | xargs kill || echo \"Failed to kill process with PID: ${driver_PID}, it may not be valid.\""
       fi
       {
         echo "Driver PID: Requested termination of ${driver_PID} and children on ${driver_HOST}"
@@ -141,7 +144,7 @@ pr_list=""
 if [[ -f "${pr_list_dbfile}" ]]; then
   pr_list=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --dbfile "${pr_list_dbfile}" --list Open Ready) || true
 fi
-if [[ -z "${pr_list+x}" ]]; then
+if [[ -z "${pr_list}" ]]; then
   echo "no PRs open and ready for checkout/build .. exiting"
   exit 0
 fi
@@ -155,7 +158,7 @@ fi
 for pr in ${pr_list}; do
   # Skip pr's that are currently Building for when overlapping driver scripts are being called from within cron
   pr_building=$("${ROOT_DIR}/ci/scripts/utils/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | grep Building) || true
-  if [[ -z "${pr_building+x}" ]]; then
+  if [[ -n "${pr_building}" ]]; then
       continue
   fi
   id=$("${GH}" pr view "${pr}" --repo "${REPO_URL}" --json id --jq '.id')

jobs/JGFS_ATMOS_FBWIND

@@ -6,7 +6,7 @@
 # GFS FBWIND PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "fbwind" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "fbwind" -c "base fbwind"
 
 ###################################
 # Specify NET and RUN Name and model

jobs/JGFS_ATMOS_GEMPAK_META

@@ -6,7 +6,7 @@
 # GFS GEMPAK META PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base gempak"
 
 
 ###############################################

jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_spec" -c "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_spec" -c "base gempak"
 
 ############################################
 # Set up model and cycle specific variables

jobs/JGLOBAL_CLEANUP

@@ -5,13 +5,13 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "cleanup" -c "base cleanup"
 
 "${SCRgfs}/exglobal_cleanup.sh"
 status=$?
-[[ ${status} -ne 0 ]] && exit "${status}"
+(( status != 0 )) && exit "${status}"
 
 ##########################################
 # Remove the Temporary working directory
 ##########################################
-cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
-[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+# DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# is removed in exglobal_cleanup.sh, nothing to do here.
 
 exit 0
 

jobs/rocoto/arch_test.sh

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+###############################################################
+exit 0

jobs/rocoto/prepoceanobs.sh

@@ -14,8 +14,10 @@ export jobid="${job}.$$"
 
 ###############################################################
 # setup python path for class defs and utils
-
-PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush"
+# shellcheck disable=SC2311
+pyiodaPATH="${HOMEgfs}/sorc/gdas.cd/build/lib/python$(detect_py_ver)/"
+PYTHONPATH="${pyiodaPATH}:${PYTHONPATH}"
+export PYTHONPATH
 
 ###############################################################
 # Execute the JJOB

parm/config/gefs/config.arch

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+########## config.arch ##########
+# Archive specific
+
+echo "BEGIN: config.arch"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" arch
+
+export ARCH_GAUSSIAN="YES"
+export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS}
+export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
+
+echo "END: config.arch"

parm/config/gefs/config.base

@@ -97,7 +97,6 @@ export PSLOT="@PSLOT@"
 export EXPDIR="@EXPDIR@/${PSLOT}"
 export ROTDIR="@COMROOT@/${PSLOT}"
 
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 

parm/config/gefs/config.resources

@@ -280,6 +280,13 @@ case ${step} in
     export is_exclusive=False
     ;;
 
+  "arch")
+    export walltime="06:00:00"
+    export ntasks=1
+    export tasks_per_node=1
+    export threads_per_task=1
+    export memory="4096M"
+    ;;
   *)
     echo "FATAL ERROR: Invalid job ${step} passed to ${BASH_SOURCE[0]}"
     exit 1

parm/config/gfs/config.base

@@ -127,7 +127,6 @@ export DUMP_SUFFIX=""
 if [[ "${PDY}${cyc}" -ge "2019092100" && "${PDY}${cyc}" -le "2019110700" ]]; then
     export DUMP_SUFFIX="p"              # Use dumps from NCO GFS v15.3 parallel
 fi
-export DATAROOT="${STMP}/RUNDIRS/${PSLOT}"  # TODO: set via prod_envir in Ops
 export ARCDIR="${NOSCRUB}/archive/${PSLOT}"
 export ATARDIR="@ATARDIR@"
 

parm/config/gfs/config.fbwind

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.gempak ##########
+# GFS fbwind step specific
+
+echo "BEGIN: config.fbwind"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" fbwind
+
+echo "END: config.fbwind"

parm/config/gfs/config.resources

@@ -544,7 +544,7 @@ case ${step} in
     case ${OCNRES} in
       "025")
         memory="128GB"
-        ntasks=40;;
+        ntasks=16;;
       "050")
         memory="32GB"
         ntasks=16;;
@@ -1124,6 +1124,13 @@ case ${step} in
     memory_gfs="2GB"
     ;;
 
+  "fbwind")
+    walltime="00:05:00"
+    ntasks=1
+    threads_per_task=1
+    memory="4GB"
+    ;;
+
   "mos_stn_prep")
     walltime="00:10:00"
     ntasks=3

parm/gdas/staging/atm_lgetkf_bkg.yaml.j2

@@ -1,6 +1,3 @@
-{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
-{% set time_list = [current_cycle] %}
-
 mkdir:
 {% for imem in range(1,NMEM_ENS+1) %}
    {% set memchar = 'mem%03d' | format(imem) %}
@@ -14,19 +11,13 @@ mkdir:
 - '{{ COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) }}'
 {% endfor %}
 copy:
-{% for time in time_list %}
-   {% for imem in range(1,NMEM_ENS+1) %}
-      {% set memchar = 'mem%03d' | format(imem) %}
-      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
-      	      	      	    '${RUN}': 'enkfgdas',
-                            '${YMD}': previous_cycle | to_YMD,	
-                            '${HH}': previous_cycle | strftime('%H'),
-                            '${MEMDIR}': memchar }) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/{{ memchar }}/']
-      {% for ftype in ftype_list %}
-         {% for itile in range(1,7) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/{{ memchar }}/']
-         {% endfor %}
-      {% endfor %}
-   {% endfor %}
+{% for imem in range(1,NMEM_ENS+1) %}
+   {% set memchar = 'mem%03d' | format(imem) %}
+   {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                         '${RUN}': 'enkfgdas',
+                         '${YMD}': previous_cycle | to_YMD,	
+                         '${HH}': previous_cycle | strftime('%H'),
+                         '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/{{ memchar }}/']
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/{{ memchar }}/']
 {% endfor %}

parm/gdas/staging/atm_var_bkg.yaml.j2

@@ -1,14 +1,5 @@
-{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
-{% set time_list = [current_cycle] %}
-
 mkdir:
 - '{{ DATA }}/bkg'
 copy:
-{% for time in time_list %}
-- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/bkg/']
-   {% for ftype in ftype_list %}
-      {% for itile in range(1,ntiles+1) %}
-- ['{{ COM_ATMOS_RESTART_PREV }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/bkg/']
-      {% endfor %}
-   {% endfor %}
-{% endfor %}
+- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/bkg/']
+- ['{{ COM_ATMOS_HISTORY_PREV }}/{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/bkg/']

parm/gdas/staging/atm_var_fv3ens.yaml.j2

@@ -1,24 +1,15 @@
-{% set ftype_list = ['fv_core.res', 'fv_srf_wnd.res', 'fv_tracer.res', 'phy_data', 'sfc_data'] %}
-{% set time_list = [current_cycle] %}
-
 mkdir:
 {% for imem in range(1,NMEM_ENS+1) %}
 - '{{ DATA }}/ens/{{ 'mem%03d' | format(imem) }}'
 {% endfor %}
 copy:
-{% for time in time_list %}
-   {% for imem in range(1,NMEM_ENS+1) %}
-      {% set memchar = 'mem%03d' | format(imem) %}
-      {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
-                            '${RUN}': 'enkfgdas',
-                            '${YMD}': previous_cycle | to_YMD,
-                            '${HH}': previous_cycle | strftime('%H'),
-                            '${MEMDIR}': memchar }) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.coupler.res', '{{ DATA }}/ens/{{ memchar }}/']
-      {% for ftype in ftype_list %}
-         {% for itile in range(1,ntiles+1) %}
-- ['{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ time | to_fv3time }}.{{ ftype }}.tile{{ itile }}.nc', '{{ DATA }}/ens/{{ memchar }}/']
-         {% endfor %}
-      {% endfor %}
-   {% endfor %}
+{% for imem in range(1,NMEM_ENS+1) %}
+   {% set memchar = 'mem%03d' | format(imem) %}
+   {% set tmpl_dict = ({ '${ROTDIR}': ROTDIR,
+                         '${RUN}': 'enkfgdas',
+                         '${YMD}': previous_cycle | to_YMD,
+                         '${HH}': previous_cycle | strftime('%H'),
+                         '${MEMDIR}': memchar }) %}
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_atmf006.nc', '{{ DATA }}/ens/{{ memchar }}/']
+- ['{{ COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) }}/enkf{{ GPREFIX }}cubed_sphere_grid_sfcf006.nc', '{{ DATA }}/ens/{{ memchar }}/']
 {% endfor %}

scripts/exglobal_cleanup.sh

@@ -12,41 +12,26 @@ if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 
-# Search and delete files/directories from DATAROOT/ older than ${purge_every_days} days
-# purge_every_days should be a positive integer
-#purge_every_days=3
-
-# Find and delete files older than ${purge_every_days} days
-#find "${DATAROOT}/"* -type f -mtime "+${purge_every_days}" -exec rm -f {} \;
-
-# Find and delete directories older than ${purge_every_days} days
-#find "${DATAROOT}/"* -type d -mtime "+${purge_every_days}" -exec rm -rf {} \;
+# In XML, DATAROOT is defined as:
+#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# cleanup is only executed after the entire cycle is successfully completed.
+# removing DATAROOT should be possible if that is the case.
+rm -rf "${DATAROOT}"
 
 echo "Cleanup ${DATAROOT} completed!"
 ###############################################################
 
-###############################################################
-# Clean up previous cycles; various depths
-# PRIOR CYCLE: Leave the prior cycle alone
-# shellcheck disable=SC2153
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${assim_freq} hours")
-# PREVIOUS to the PRIOR CYCLE
-GDATE=$(date --utc +%Y%m%d%H -d "${GDATE:0:8} ${GDATE:8:2} -${assim_freq} hours")
-
-# Remove the TMPDIR directory
-# TODO Only prepbufr is currently using this directory, and all jobs should be
-#   cleaning up after themselves anyway
-COMIN="${DATAROOT}/${GDATE}"
-[[ -d ${COMIN} ]] && rm -rf "${COMIN}"
-
 if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
     exit 0
 fi
 
+###############################################################
+# Clean up previous cycles; various depths
+
 # Step back every assim_freq hours and remove old rotating directories
 # for successful cycles (defaults from 24h to 120h).
 # Retain files needed by Fit2Obs
-last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours" )
+last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours")
 first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
 last_rtofs=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDRTOFS:-48} hours")
 function remove_files() {