Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable traditional threading as an option #3149

Open
wants to merge 5 commits into
base: develop
Choose a base branch
from
+95 −33
Open

Enable traditional threading as an option #3149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cf21059
compute resources when doing traditional threading
aerorahul Dec 10, 2024
d5335a1
tasks_per_node should know about threads
aerorahul Dec 10, 2024
bdce86f
spacing
aerorahul Dec 10, 2024
0ee666d
add QUILTING_RESTART as an option
aerorahul Dec 10, 2024
26a0c42
update config.resources in GEFS system to allow use of traditional th…
aerorahul Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 47 additions & 16 deletions parm/config/gefs/config.resources
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,29 +110,41 @@ case ${step} in
[[ "${DO_ICE}" == "YES" ]] && nthreads_cice6=1
fi

# PETS for the atmosphere dycore
(( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"
# FV3
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( FV3THREADS = nthreads_fv3 ))
(( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
else
(( FV3THREADS = UFS_THREADS ))
(( FV3PETS = ntasks_fv3 ))
fi
echo "FV3 using (nthreads, PETS) = (${FV3THREADS}, ${FV3PETS})"

# PETS for quilting
# Write grid component
QUILTPETS=0; QUILTTHREADS=0
if [[ "${QUILTING:-}" == ".true." ]]; then
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( QUILTTHREADS = nthreads_fv3 ))
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
else
(( QUILTTHREADS = UFS_THREADS ))
(( QUILTPETS = ntasks_quilt ))
fi
(( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD ))
export WRTTASK_PER_GROUP
else
QUILTPETS=0
fi
echo "QUILT using (nthreads, PETS) = (${nthreads_fv3}, ${QUILTPETS})"
echo "QUILT using (nthreads, PETS) = (${QUILTTHREADS}, ${QUILTPETS})"

# Total PETS for the atmosphere component
ATMTHREADS=${nthreads_fv3}
ATMTHREADS=${FV3THREADS}
(( ATMPETS = FV3PETS + QUILTPETS ))
export ATMPETS ATMTHREADS
echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})"

# Total PETS for the coupled model (starting w/ the atmosphere)
NTASKS_TOT=${ATMPETS}

# Mediator
# The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks.
# However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance.
# See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit
Expand All @@ -143,6 +155,7 @@ case ${step} in
export MEDPETS MEDTHREADS
echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})"

# GOCART
CHMPETS=0; CHMTHREADS=0
if [[ "${DO_AERO_FCST}" == "YES" ]]; then
# GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks).
Expand All @@ -153,28 +166,46 @@ case ${step} in
fi
export CHMPETS CHMTHREADS

# Waves
WAVPETS=0; WAVTHREADS=0
if [[ "${DO_WAVE}" == "YES" ]]; then
(( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
(( WAVTHREADS = nthreads_ww3 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( WAVTHREADS = nthreads_ww3 ))
(( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
else
(( WAVTHREADS = UFS_THREADS ))
(( WAVPETS = ntasks_ww3 ))
fi
echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})"
(( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
fi
export WAVPETS WAVTHREADS

# Ocean
OCNPETS=0; OCNTHREADS=0
if [[ "${DO_OCN}" == "YES" ]]; then
(( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
(( OCNTHREADS = nthreads_mom6 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( OCNTHREADS = nthreads_mom6 ))
(( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
else
(( OCNTHREADS = UFS_THREADS ))
(( OCNPETS = ntasks_mom6 ))
fi
echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})"
(( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
fi
export OCNPETS OCNTHREADS

# Ice
ICEPETS=0; ICETHREADS=0
if [[ "${DO_ICE}" == "YES" ]]; then
(( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
(( ICETHREADS = nthreads_cice6 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( ICETHREADS = nthreads_cice6 ))
(( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
else
(( ICETHREADS = UFS_THREADS ))
(( ICEPETS = ntasks_cice6 ))
fi
echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})"
(( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
fi
Expand All @@ -184,7 +215,7 @@ case ${step} in

declare -x "ntasks"="${NTASKS_TOT}"
declare -x "threads_per_task"="${UFS_THREADS}"
declare -x "tasks_per_node"="${max_tasks_per_node}"
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))

case "${CASE}" in
"C48" | "C96" | "C192")
Expand Down
63 changes: 47 additions & 16 deletions parm/config/gfs/config.resources
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -775,29 +775,41 @@ case ${step} in
export npy_nest=${npy_nest:-961}
fi

# PETS for the atmosphere dycore
(( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
echo "FV3 using (nthreads, PETS) = (${nthreads_fv3}, ${FV3PETS})"
# FV3
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( FV3THREADS = nthreads_fv3 ))
(( FV3PETS = ntasks_fv3 * nthreads_fv3 ))
else
(( FV3THREADS = UFS_THREADS ))
(( FV3PETS = ntasks_fv3 ))
fi
echo "FV3 using (nthreads, PETS) = (${FV3THREADS}, ${FV3PETS})"

# PETS for quilting
# Write grid component
QUILTPETS=0; QUILTTHREADS=0
if [[ "${QUILTING:-}" == ".true." ]]; then
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( QUILTTHREADS = nthreads_fv3 ))
(( QUILTPETS = ntasks_quilt * nthreads_fv3 ))
else
(( QUILTTHREADS = UFS_THREADS ))
(( QUILTPETS = ntasks_quilt ))
fi
(( WRTTASK_PER_GROUP = WRTTASK_PER_GROUP_PER_THREAD ))
export WRTTASK_PER_GROUP
else
QUILTPETS=0
fi
echo "QUILT using (nthreads, PETS) = (${nthreads_fv3}, ${QUILTPETS})"
echo "QUILT using (nthreads, PETS) = (${QUILTTHREADS}, ${QUILTPETS})"

# Total PETS for the atmosphere component
ATMTHREADS=${nthreads_fv3}
ATMTHREADS=${FV3THREADS}
(( ATMPETS = FV3PETS + QUILTPETS ))
export ATMPETS ATMTHREADS
echo "FV3ATM using (nthreads, PETS) = (${ATMTHREADS}, ${ATMPETS})"

# Total PETS for the coupled model (starting w/ the atmosphere)
NTASKS_TOT=${ATMPETS}

# Mediator
# The mediator PETS can overlap with other components, usually it lands on the atmosphere tasks.
# However, it is suggested limiting mediator PETS to 300, as it may cause the slow performance.
# See https://docs.google.com/document/d/1bKpi-52t5jIfv2tuNHmQkYUe3hkKsiG_DG_s6Mnukog/edit
Expand All @@ -808,6 +820,7 @@ case ${step} in
export MEDPETS MEDTHREADS
echo "MEDIATOR using (threads, PETS) = (${MEDTHREADS}, ${MEDPETS})"

# GOCART
CHMPETS=0; CHMTHREADS=0
if [[ "${DO_AERO_FCST}" == "YES" ]]; then
# GOCART shares the same grid and forecast tasks as FV3 (do not add write grid component tasks).
Expand All @@ -818,28 +831,46 @@ case ${step} in
fi
export CHMPETS CHMTHREADS

# Waves
WAVPETS=0; WAVTHREADS=0
if [[ "${DO_WAVE}" == "YES" ]]; then
(( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
(( WAVTHREADS = nthreads_ww3 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( WAVTHREADS = nthreads_ww3 ))
(( WAVPETS = ntasks_ww3 * nthreads_ww3 ))
else
(( WAVTHREADS = UFS_THREADS ))
(( WAVPETS = ntasks_ww3 ))
fi
echo "WW3 using (threads, PETS) = (${WAVTHREADS}, ${WAVPETS})"
(( NTASKS_TOT = NTASKS_TOT + WAVPETS ))
fi
export WAVPETS WAVTHREADS

# Ocean
OCNPETS=0; OCNTHREADS=0
if [[ "${DO_OCN}" == "YES" ]]; then
(( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
(( OCNTHREADS = nthreads_mom6 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( OCNTHREADS = nthreads_mom6 ))
(( OCNPETS = ntasks_mom6 * nthreads_mom6 ))
else
(( OCNTHREADS = UFS_THREADS ))
(( OCNPETS = ntasks_mom6 ))
fi
echo "MOM6 using (threads, PETS) = (${OCNTHREADS}, ${OCNPETS})"
(( NTASKS_TOT = NTASKS_TOT + OCNPETS ))
fi
export OCNPETS OCNTHREADS

# Ice
ICEPETS=0; ICETHREADS=0
if [[ "${DO_ICE}" == "YES" ]]; then
(( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
(( ICETHREADS = nthreads_cice6 ))
if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
(( ICETHREADS = nthreads_cice6 ))
(( ICEPETS = ntasks_cice6 * nthreads_cice6 ))
else
(( ICETHREADS = UFS_THREADS ))
(( ICEPETS = ntasks_cice6 ))
fi
echo "CICE6 using (threads, PETS) = (${ICETHREADS}, ${ICEPETS})"
(( NTASKS_TOT = NTASKS_TOT + ICEPETS ))
fi
Expand All @@ -849,7 +880,7 @@ case ${step} in

declare -x "ntasks"="${NTASKS_TOT}"
declare -x "threads_per_task"="${UFS_THREADS}"
declare -x "tasks_per_node"="${max_tasks_per_node}"
tasks_per_node=$(( max_tasks_per_node / threads_per_task ))

case "${CASE}" in
"C48" | "C96" | "C192")
Expand Down
2 changes: 1 addition & 1 deletion ush/parsing_model_configure_FV3.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ local FHROT=${IAU_FHROT:-0}
local DT_ATMOS=${DELTIM}
local RESTART_INTERVAL="${FV3_RESTART_FH[*]}"
# QUILTING
local QUILTING_RESTART=".true."
local QUILTING_RESTART="${QUILTING_RESTART:-${QUILTING}}"
local WRITE_GROUP=${WRITE_GROUP:-1}
local WRTTASK_PER_GROUP=${WRTTASK_PER_GROUP:-24}
local ITASKS=1
Expand Down