Skip to content

Commit

Permalink
Updates from RRFS_A v0.7.3 retro tests (#93)
Browse files Browse the repository at this point in the history
* Try to use multiple threads to run model.

* update srun in prep_cyc for JET and Hera.

* Update RRFS_A configuration for retros.
Set tg3_from_soil=True for make_ics.

---------

Co-authored-by: ming hu <[email protected]>
  • Loading branch information
hu5970 and ming hu authored Nov 7, 2023
1 parent 2527f00 commit d3bf3fd
Show file tree
Hide file tree
Showing 11 changed files with 258 additions and 35 deletions.
5 changes: 5 additions & 0 deletions parm/model_configure
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,14 @@ fhrot: {{ fhrot }}
RUN_CONTINUE: .false.
ENS_SPS: .false.
dt_atmos: {{ dt_atmos }}
cpl: .false.
calendar: 'julian'
memuse_verbose: .false.
atmos_nthreads: {{ atmos_nthreads }}
use_hyper_thread: .false.
debug_affinity: .true.
restart_interval: {{ restart_interval }}
use_hyper_thread: .false.
output_1st_tstep_rst: .false.
write_dopost: {{ write_dopost }}
ideflate: {{ ideflate }}
Expand Down
8 changes: 0 additions & 8 deletions parm/nems.configure
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,6 @@
#### NEMS Run-time Configuration File #####
#############################################

# ESMF #
{%- if print_esmf %}
logKindFlag: ESMF_LOGKIND_MULTI
{%- else %}
logKindFlag: ESMF_LOGKIND_MULTI_ON_ERROR
{%- endif %}
globalResourceControl: true

# EARTH #
EARTH_component_list: ATM
# ATM #
Expand Down
4 changes: 2 additions & 2 deletions scripts/exrrfs_make_ics.sh
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,7 @@ case "${EXTRN_MDL_NAME_ICS}" in
tracers="[\"sphum\",\"liq_wat\",\"o3mr\",\"ice_wat\",\"rainwat\",\"snowwat\",\"graupel\"]"
external_model="FV3GFS"
input_type="gaussian_netcdf"
convert_nst=True
convert_nst=False
fn_atm="${EXTRN_MDL_FNS[0]}"
fn_sfc="${EXTRN_MDL_FNS[1]}"
fi
Expand All @@ -415,7 +415,7 @@ case "${EXTRN_MDL_NAME_ICS}" in
vgfrc_from_climo=True
minmax_vgfrc_from_climo=True
lai_from_climo=True
tg3_from_soil=False
tg3_from_soil=True
;;

"GDASENKF")
Expand Down
7 changes: 4 additions & 3 deletions scripts/exrrfs_run_fcst.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ ulimit -a
case $MACHINE in

"WCOSS2")
export OMP_STACKSIZE=1G
OMP_NUM_THREADS=${TPP_RUN_FCST}
OMP_STACKSIZE=1G
export MPICH_ABORT_ON_ERROR=1
export MALLOC_MMAP_MAX_=0
export MALLOC_TRIM_THRESHOLD_=134217728
Expand All @@ -101,7 +102,7 @@ case $MACHINE in
export MPICH_OFI_STARTUP_CONNECT=1
export MPICH_OFI_VERBOSE=1
export MPICH_OFI_NIC_VERBOSE=1
APRUN="mpiexec -n ${PE_MEMBER01} -ppn ${PPN_RUN_FCST} --cpu-bind core --depth ${TPP_RUN_FCST}"
APRUN="mpiexec -n ${PE_MEMBER01} -ppn ${PPN_RUN_FCST} --cpu-bind core --depth ${OMP_NUM_THREADS}"
;;

"HERA")
Expand All @@ -128,7 +129,7 @@ esac
#-----------------------------------------------------------------------
#
export KMP_AFFINITY=scatter
export OMP_NUM_THREADS=${TPP_RUN_FCST:-1}
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
export OMP_STACKSIZE=${OMP_STACKSIZE:-1024m}
#
#-----------------------------------------------------------------------
Expand Down
9 changes: 5 additions & 4 deletions scripts/exrrfs_run_prepstart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -77,19 +77,20 @@ print_input_args valid_args
case $MACHINE in

"WCOSS2")
APRUN="time"
ncores=$(( NNODES_RUN_PREPSTART*PPN_RUN_PREPSTART))
APRUN="mpiexec -n ${ncores} -ppn ${PPN_RUN_PREPSTART}"
;;

"HERA")
APRUN="time"
APRUN="srun --export=ALL --mem=0"
;;

"ORION")
APRUN="time"
APRUN="srun --export=ALL"
;;

"JET")
APRUN="time"
APRUN="srun --export=ALL --mem=0"
;;

*)
Expand Down
1 change: 1 addition & 0 deletions ush/create_model_configure_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ def create_model_configure_file(
"nhours_fcst": FCST_LEN_HRS_thiscycle,
"fhrot": fhrot,
"dt_atmos": DT_ATMOS,
'atmos_nthreads': nthreads,
"restart_interval": restart_interval,
"write_dopost": dot_write_dopost,
"quilting": dot_quilting_dot,
Expand Down
1 change: 1 addition & 0 deletions ush/link_fix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ Creating links in the FIXLAM directory to the grid files..."
#
"grid")
fns=( \
"C*${DOT_OR_USCORE}fvcom_mask.nc" \
"C*${DOT_OR_USCORE}mosaic.halo${NHW}.nc" \
"C*${DOT_OR_USCORE}mosaic.halo${NH4}.nc" \
"C*${DOT_OR_USCORE}mosaic.halo${NH3}.nc" \
Expand Down
14 changes: 7 additions & 7 deletions ush/sample_configs/RRFS_A/config.sh_rrfs_a_enkf_n3
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MACHINE="wcoss2"
MACHINETYPE="primary"
version="v0.7.1"
version="v0.7.3"
ACCOUNT="RRFS_DEV"
#RESERVATION="rrfsdet"
EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}"
Expand Down Expand Up @@ -50,8 +50,8 @@ BOUNDARY_PROC_GROUP_NUM="12"

# avaialble retro period:
# 20210511-20210531; 20210718-20210801
DATE_FIRST_CYCL="20231001"
DATE_LAST_CYCL="20231031"
DATE_FIRST_CYCL="20231101"
DATE_LAST_CYCL="20231130"
CYCL_HRS=( "00" "12" )
CYCL_HRS=( "18" )
CYCL_HRS_SPINSTART=("06" "18")
Expand All @@ -60,7 +60,7 @@ if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
CYCL_HRS_STOCH=("00" "06" "12" "18")
fi
#CYCL_HRS_RECENTER=("19")
CYCLEMONTH="10"
CYCLEMONTH="11"
CYCLEDAY="*"

STARTYEAR=${DATE_FIRST_CYCL:0:4}
Expand Down Expand Up @@ -189,10 +189,10 @@ fi

RUN_ensctrl="rrfs"
RUN="enkfrrfs"
TAG="n3enkf71"
TAG="n3enkf73"
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
RUN="refs"
TAG="n3enfcst71"
TAG="n3enfcst73"
fi
COMINgfs=""

Expand All @@ -202,7 +202,7 @@ EXTRN_MDL_SOURCE_BASEDIR_ICS=/lfs/h1/ops/prod/com/gefs/v12.3
EXTRN_MDL_SOURCE_BASEDIR_LBCS=/lfs/h1/ops/prod/com/gefs/v12.3

STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files.
PTMP="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files.
PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files.
NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/enfcst" # Path to directory STMP that mostly contains input files.
Expand Down
210 changes: 210 additions & 0 deletions ush/sample_configs/RRFS_A/config.sh_rrfs_a_enkf_n3_retro
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
MACHINE="wcoss2"
MACHINETYPE="backup"
version="v0.7.3"
ACCOUNT="RRFS_DEV"
#RESERVATION="rrfsdet"
EXPT_BASEDIR="/lfs/h2/emc/lam/noscrub/emc.lam/rrfs/${version}"
EXPT_SUBDIR="rrfs_na_enkf"

PREDEF_GRID_NAME="RRFS_NA_3km"

. set_rrfs_config_general.sh

DO_ENSEMBLE="TRUE"
#DO_ENSFCST="TRUE"
#DO_DACYCLE="TRUE"
#DO_SURFACE_CYCLE="TRUE"
DO_SPINUP="TRUE"
DO_SAVE_DA_OUTPUT="TRUE"
DO_POST_SPINUP="FALSE"
DO_POST_PROD="FALSE"
DO_RETRO="TRUE"
DO_NONVAR_CLDANAL="TRUE"
#DO_ENVAR_RADAR_REF="TRUE"
DO_SMOKE_DUST="FALSE"
#DO_REFL2TTEN="FALSE"
#RADARREFL_TIMELEVEL=(0)
#FH_DFI_RADAR="0.0,0.25,0.5"
#DO_SOIL_ADJUST="TRUE"
#DO_RADDA="FALSE"
#DO_BUFRSND="TRUE"
#USE_FVCOM="TRUE"
#PREP_FVCOM="TRUE"
USE_CLM="TRUE"
DO_PARALLEL_PRDGEN="TRUE"
DO_GSIDIAG_OFFLINE="FALSE"

if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
EXPT_SUBDIR="rrfs_na_enfcst"
DO_SPINUP="FALSE"
DO_SAVE_DA_OUTPUT="FALSE"
DO_NONVAR_CLDANAL="FALSE"
DO_POST_PROD="TRUE"
fi

EXTRN_MDL_ICS_OFFSET_HRS="30"
LBC_SPEC_INTVL_HRS="1"
EXTRN_MDL_LBCS_OFFSET_HRS="6"
BOUNDARY_LEN_HRS="12"
BOUNDARY_PROC_GROUP_NUM="12"

# avaialble retro period:
# 20210511-20210531; 20210718-20210801
DATE_FIRST_CYCL="20220720"
DATE_LAST_CYCL="20220720"
CYCL_HRS=( "00" "12" )
CYCL_HRS=( "18" )
CYCL_HRS_SPINSTART=("06" "18")
CYCL_HRS_PRODSTART=("07" "19")
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
CYCL_HRS_STOCH=("00" "06" "12" "18")
fi
#CYCL_HRS_RECENTER=("19")
CYCLEMONTH="07"
CYCLEDAY="20"

STARTYEAR=${DATE_FIRST_CYCL:0:4}
STARTMONTH=${DATE_FIRST_CYCL:4:2}
STARTDAY=${DATE_FIRST_CYCL:6:2}
STARTHOUR="00"
ENDYEAR=${DATE_LAST_CYCL:0:4}
ENDMONTH=${DATE_LAST_CYCL:4:2}
ENDDAY=${DATE_LAST_CYCL:6:2}
ENDHOUR="23"

PREEXISTING_DIR_METHOD="upgrade" # "rename"
INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0600 ${DATE_LAST_CYCL}2300 12:00:00"
BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0600 ${DATE_LAST_CYCL}2300 06:00:00"
PROD_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 01:00:00"
PRODLONG_CYCLEDEF="00 01 01 01 2100 *"
#RECENTER_CYCLEDEF="00 19 * 10 2022 *"
#ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1400 ${DATE_LAST_CYCL}2300 24:00:00"
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
BOUNDARY_LEN_HRS="60"
LBC_SPEC_INTVL_HRS="3"
DO_SPINUP="FALSE"
INITIAL_CYCLEDEF="00 01 01 01 2100 *"
BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
PROD_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
RECENTER_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
fi
if [[ $DO_SPINUP == "TRUE" ]] ; then
SPINUP_CYCLEDEF="${DATE_FIRST_CYCL}0600 ${DATE_LAST_CYCL}2300 12:00:00"
fi
if [[ $DO_SAVE_DA_OUTPUT == "TRUE" ]] ; then
SAVEDA_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
fi

FCST_LEN_HRS="1"
FCST_LEN_HRS_SPINUP="1"
POSTPROC_LEN_HRS="1"
#FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18)
for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=1; done
for i in {0..23..6}; do FCST_LEN_HRS_CYCLES[$i]=1; done
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
for i in {0..23..06}; do FCST_LEN_HRS_CYCLES[$i]=60; done
FCST_LEN_HRS="60"
POSTPROC_LEN_HRS="60"
BOUNDARY_PROC_GROUP_NUM="10"
fi
DA_CYCLE_INTERV="1"
RESTART_INTERVAL="1"
RESTART_INTERVAL_LONG="1"
netcdf_diag=.true.
binary_diag=.false.
WRTCMP_output_file="netcdf_parallel"

## set up post
#NFHOUT_HF="1"
#NFHMAX_HF="18"
#NFHOUT="3"

WTIME_RUN_FCST="00:30:00"
WTIME_RUN_FCST_LONG="04:45:00"
PPN_RUN_ANAL="16"
NNODES_RUN_ANAL="1"
TPP_RUN_ANAL="8"

EXTRN_MDL_NAME_ICS="GEFS"
EXTRN_MDL_NAME_LBCS="GEFS"
FV3GFS_FILE_FMT_ICS="grib2"
FV3GFS_FILE_FMT_LBCS="grib2"
EXTRN_MDL_SAVETYPE="GSL"

envir="para"
NET="rrfs_a"
ARCHIVEDIR="/NCEPDEV/emc-meso/1year/emc.lam"
NCL_REGION="na"
MODEL="RRFS_A"

if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then
NUM_ENS_MEMBERS=30
# DO_ENSCONTROL="TRUE"
DO_GSIOBSERVER="TRUE"
DO_ENKFUPDATE="TRUE"
# DO_RECENTER="TRUE"
DO_ENS_GRAPHICS="TRUE"
DO_ENKF_RADAR_REF="TRUE"
DO_ENSPOST="FALSE"
DO_ENSINIT="TRUE"

RADAR_REF_THINNING="2"
ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens"
CLEAN_OLDFCST_HRS="12"
CLEAN_OLDSTMPPOST_HRS="12"
cld_bld_hgt=0.0
l_precip_clear_only=.true.
write_diag_2=.true.

START_TIME_SPINUP="00:30:00"
LAYOUT_X="15"
LAYOUT_Y="55"
NNODES_RUN_FCST="13"
WRTCMP_write_tasks_per_group="7"

NUM_ENS_MEMBERS_FCST=5
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
NUM_ENS_MEMBERS=${NUM_ENS_MEMBERS_FCST}
WTIME_RUN_FCST="04:45:00"
WTIME_MAKE_LBCS="01:30:00"

LAYOUT_X="50"
LAYOUT_Y="64"
NNODES_RUN_FCST="51"
WRTCMP_write_tasks_per_group="64"

DO_SPP="TRUE"
DO_SPPT="FALSE"
DO_SKEB="FALSE"
SPPT_MAG="0.5"
DO_LSM_SPP="TRUE"
DO_RECENTER="TRUE"

fi
# PPN_RUN_RECENTER="$(( ${NUM_ENS_MEMBERS} + 1 ))"
# NNODES_RUN_RECENTER="3"
PPN_RUN_RECENTER="128"
NNODES_RUN_RECENTER="10"
fi

RUN_ensctrl="rrfs"
RUN="enkfrrfs"
TAG="n3enkf73"
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
RUN="refs"
TAG="n3enfcst73"
fi
COMINgfs=""

. set_rrfs_config.sh

STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files.
PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains input files.
NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files
if [[ ${DO_ENSFCST} == "TRUE" ]] ; then
STMP="/lfs/h2/emc/stmp/emc.lam/rrfs/${version}/enfcst" # Path to directory STMP that mostly contains input files.
NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory NWGES that save boundary, cold initial, restart files
fi
ENSCTRL_STMP="/lfs/f2/t2o/ptmp/emc/stmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering.
ENSCTRL_PTMP="/lfs/h2/emc/ptmp/emc.lam/rrfs/${version}" # Path to directory STMP that mostly contains control input files for ensemble recentering.
ENSCTRL_NWGES="/lfs/f2/t2o/ptmp/emc/ptmp/emc.lam/rrfs/${version}/nwges" # Path to directory STMP that mostly contains control input files for ensemble recentering.
Loading

0 comments on commit d3bf3fd

Please sign in to comment.