Fix wave restarts and GEFS FHOUT/FHMAX (NOAA-EMC#3009)

# Description
Fixes some issues that were preventing wave restarts from operating
correctly.

First, the wave restart files were not being correctly linked from
`$DATA` to `$DATArestart`. The files are placed in the root of `$DATA`
instead of in `${DATA}/WAVE_RESTART`, so now links for the individual
files are created.

Second, the incorrect filenames were being searches for and copied as
part of a rerun. Filenames were geared towards multigrid waves, which
use the grid names, but single grid just uses a `ww3`. Since multigrid
waves are deprecated in workflow and will soon be removed (NOAA-EMC#2637), these
were updated only supporting the single-grid option.

These fixes allow forecast segments (and emergency restarts) to work
correctly when waves are on.

Additionally, the `FHMAX` (and `FHOUT`) for perturbed GEFS members was
being overwritten by `config.efcs` due to remnant code from when it was
copied from the EnKF version. This interfered with the segment setting
for those GEFS members.

Resolves NOAA-EMC#3001

# Type of change
- [x] Bug fix (fixes something broken)
- [ ] New feature (adds functionality)
- [ ] Maintenance (code refactor, clean-up, new CI test, etc.)

# Change characteristics
- Is this a breaking change (a change in existing functionality)? NO
- Does this change require a documentation update? NO
- Does this change require an update to any of the following submodules?
NO

# How has this been tested?
- S2SW forecast-only test with segments on Hercules

# Checklist
- [x] Any dependent changes have been merged and published
- [x] My code follows the style guidelines of this project
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have documented my code, including function, input, and output
descriptions
- [x] My changes generate no new warnings
- [x] New and existing tests pass with my changes
- [x] This change is covered by an existing CI test or a new one has
been added
- [x] I have made corresponding changes to the system documentation if
necessary

---------

Co-authored-by: Kate.Friedman <[email protected]>
Co-authored-by: David Huber <[email protected]>

parm/config/gefs/config.efcs

@@ -26,18 +26,6 @@ source "${EXPDIR}/config.ufs" ${string}
 # Get task specific resources
 source "${EXPDIR}/config.resources" efcs
 
-# nggps_diag_nml
-export FHOUT=${FHOUT_ENKF:-3}
-if [[ "${RUN}" == "enkfgfs" ]]; then
-    export FHOUT=${FHOUT_ENKF_GFS:-${FHOUT}}
-fi
-
-# model_configure
-export FHMAX=${FHMAX_ENKF:-9}
-if [[ "${RUN}" == "enkfgfs" ]]; then
-   export FHMAX=${FHMAX_ENKF_GFS:-${FHMAX}}
-fi
-
 # Stochastic physics parameters (only for ensemble forecasts)
 export DO_SKEB="YES"
 export SKEB="0.8,-999,-999,-999,-999"
@@ -74,6 +62,6 @@ if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
 else
     export ODA_INCUPD="False"
 fi
-export restart_interval="${restart_interval_enkfgfs:-12}"
+export restart_interval="${restart_interval_gfs:-12}"
 
 echo "END: config.efcs"

parm/config/gefs/config.fcst

@@ -35,7 +35,8 @@ IFS=', ' read -ra segments <<< "${FCST_SEGMENTS}"
 # Determine MIN and MAX based on the forecast segment
 export FHMIN=${segments[${FCST_SEGMENT}]}
 export FHMAX=${segments[${FCST_SEGMENT}+1]}
-# Cap other FHMAX variables at FHMAX for the segment
+# Cap other FH variables at FHMAX for the segment
+export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV ))
 export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS ))
 export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV ))
 # shellcheck disable=SC2153

parm/config/gfs/config.fcst

@@ -38,7 +38,8 @@ case ${RUN} in
     # Determine MIN and MAX based on the forecast segment
     export FHMIN=${segments[${FCST_SEGMENT}]}
     export FHMAX=${segments[${FCST_SEGMENT}+1]}
-    # Cap other FHMAX variables at FHMAX for the segment
+    # Cap other FH variables at FHMAX for the segment
+    export FHMIN_WAV=$(( FHMIN > FHMIN_WAV ? FHMIN : FHMIN_WAV ))
     export FHMAX_HF=$(( FHMAX_HF_GFS > FHMAX ? FHMAX : FHMAX_HF_GFS ))
     export FHMAX_WAV=$(( FHMAX_WAV > FHMAX ? FHMAX : FHMAX_WAV ))
     # shellcheck disable=SC2153

parm/stage/wave.yaml.j2

@@ -9,5 +9,9 @@ wave:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}
         {% set COMOUT_WAVE_RESTART_PREV_MEM = COMOUT_WAVE_RESTART_PREV_MEM_list[imem] %}
-        - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"]
+        {% if path_exists(ICSDIR ~ "/" ~ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) ~ "/" ~ m_prefix ~ ".restart." ~ waveGRD) %}
+        - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.{{ waveGRD }}", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}/{{ m_prefix }}.restart.ww3"]
+        {% else %}
+        - ["{{ ICSDIR }}/{{ COMOUT_WAVE_RESTART_PREV_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.restart.ww3", "{{ COMOUT_WAVE_RESTART_PREV_MEM }}"]
+        {% endif %}
         {% endfor %} # mem loop

ush/forecast_det.sh

@@ -93,12 +93,9 @@ UFS_det(){
 
     # Check for WW3 restart availability
     if [[ "${cplwav}" == ".true." ]]; then
-      local ww3_grid
-      for ww3_grid in ${waveGRD} ; do
-        if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.${ww3_grid}" ]]; then
-          ww3_rst_ok="NO"
-        fi
-      done
+      if [[ ! -f "${DATArestart}/WW3_RESTART/${rdate:0:8}.${rdate:8:2}0000.restart.ww3" ]]; then
+        ww3_rst_ok="NO"
+      fi
     fi
 
     # Collective check

ush/forecast_postdet.sh

@@ -326,7 +326,7 @@ FV3_out() {
 WW3_postdet() {
   echo "SUB ${FUNCNAME[0]}: Linking input data for WW3"
 
-  local ww3_grid
+  local ww3_grid first_ww3_restart_out ww3_restart_file
   # Copy initial condition files:
   if [[ "${warm_start}" == ".true." ]]; then
     local restart_date restart_dir
@@ -338,29 +338,35 @@ WW3_postdet() {
       restart_dir="${COMIN_WAVE_RESTART_PREV}"
     fi
     echo "Copying WW3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'"
-    local ww3_restart_file
-    for ww3_grid in ${waveGRD} ; do
-      ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.${ww3_grid}"
-      if [[ ! -f "${ww3_restart_file}" ]]; then
-        echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
-        if [[ "${RERUN}" == "YES" ]]; then
-          # In the case of a RERUN, the WW3 restart file is required
-          echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!"
-          exit 1
-        fi
-      fi
-      if [[ "${waveMULTIGRID}" == ".true." ]]; then
-        ${NCP} "${ww3_restart_file}" "${DATA}/restart.${ww3_grid}" \
-        || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+    ww3_restart_file="${restart_dir}/${restart_date:0:8}.${restart_date:8:2}0000.restart.ww3"
+    if [[ -f "${ww3_restart_file}" ]]; then
+      ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \
+      || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+    else
+      if [[ "${RERUN}" == "YES" ]]; then
+        # In the case of a RERUN, the WW3 restart file is required
+        echo "FATAL ERROR: WW3 restart file '${ww3_restart_file}' not found for RERUN='${RERUN}', ABORT!"
+        exit 1
       else
-        ${NCP} "${ww3_restart_file}" "${DATA}/restart.ww3" \
-        || ( echo "FATAL ERROR: Unable to copy WW3 IC, ABORT!"; exit 1 )
+        echo "WARNING: WW3 restart file '${ww3_restart_file}' not found for warm_start='${warm_start}', will start from rest!"
       fi
-    done
+    fi
+
+    first_ww3_restart_out=$(date --utc -d "${restart_date:0:8} ${restart_date:8:2} + ${restart_interval} hours" +%Y%m%d%H)
   else  # cold start
     echo "WW3 will start from rest!"
+    first_ww3_restart_out="${model_start_date_current_cycle}"
   fi  # [[ "${warm_start}" == ".true." ]]
 
+  # Link restart files
+  local ww3_restart_file
+  # Use restart_date if it was determined above, otherwise use initialization date
+  for (( vdate = first_ww3_restart_out; vdate <= forecast_end_cycle;
+         vdate = $(date --utc -d "${vdate:0:8} ${vdate:8:2} + ${restart_interval} hours" +%Y%m%d%H) )); do
+    ww3_restart_file="${vdate:0:8}.${vdate:8:2}0000.restart.ww3"
+    ${NLN} "${DATArestart}/WW3_RESTART/${ww3_restart_file}" "${ww3_restart_file}"
+  done
+
   # Link output files
   local wavprfx="${RUN}wave${WAV_MEMBER:-}"
   if [[ "${waveMULTIGRID}" == ".true." ]]; then

ush/forecast_predet.sh

@@ -556,10 +556,10 @@ WW3_predet(){
   echo "SUB ${FUNCNAME[0]}: WW3 before run type determination"
 
   if [[ ! -d "${COMOUT_WAVE_HISTORY}" ]]; then mkdir -p "${COMOUT_WAVE_HISTORY}"; fi
-  if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}" ; fi
+  if [[ ! -d "${COMOUT_WAVE_RESTART}" ]]; then mkdir -p "${COMOUT_WAVE_RESTART}"; fi
 
-  if [[ ! -d "${DATArestart}/WAVE_RESTART" ]]; then mkdir -p "${DATArestart}/WAVE_RESTART"; fi
-  ${NLN} "${DATArestart}/WAVE_RESTART" "${DATA}/restart_wave"
+  if [[ ! -d "${DATArestart}/WW3_RESTART" ]]; then mkdir -p "${DATArestart}/WW3_RESTART"; fi
+  # Wave restarts are linked in postdet to only create links for files that will be created
 
   # Files from wave prep and wave init jobs
   # Copy mod_def files for wave grids