Merge pull request #256 from GEOS-ESM/feature/mathomp4/updates-for-milan

Updates for SCU17
GEOS-ESM · Oct 13, 2023 · 8a1f14d · 8a1f14d
2 parents 85eaba1 + f9dc73d
commit 8a1f14d
Showing 1 changed file with 55 additions and 22 deletions.
diff --git a/scripts/fv3_setup b/scripts/fv3_setup
@@ -306,29 +306,51 @@ endif
 ASKPROC:
 
 if ( $SITE == 'NCCS' ) then
-   echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
-   echo "   ${C2}sky  (Skylake)${CN}"
-   echo "   ${C2}cas  (Cascade Lake)${CN} (default)"
-   echo " "
-   set MODEL = `echo $<`
-   set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
-   if ( .$MODEL == .) then
-      set MODEL = 'cas'
-   endif
 
-   if( $MODEL != 'sky'  & \
-       $MODEL != 'cas' ) goto ASKPROC
+   set BUILT_ON_SLES15 = @BUILT_ON_SLES15@
 
-   if ($MODEL == 'sky') then
-      set NCPUS_PER_NODE = 40
-   else if ($MODEL == 'cas') then
-      # NCCS currently recommends that users do not run with
-      # 48 cores per node on SCU16 due to OS issues and
-      # recommends that CPU-intensive works run with 46 or less
-      # cores. As 45 is a multiple of 3, it's the best value
-      # that doesn't waste too much
-      #set NCPUS_PER_NODE = 48
-      set NCPUS_PER_NODE = 45
+   if ("$BUILT_ON_SLES15" == "TRUE") then
+      set DEFAULT_MODEL = 'mil'
+      echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
+      echo "   ${C2}mil  (Milan)${CN} (default)"
+      echo " "
+      set MODEL = `echo $<`
+      set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
+      if ( .$MODEL == .) then
+         set MODEL = 'mil'
+      endif
+
+      if( $MODEL != 'mil' ) goto ASKPROC
+
+      if ($MODEL == 'mil') then
+         # We save a couple processes for the kernel
+         set NCPUS_PER_NODE = 126
+      endif
+   else
+      echo "Enter the ${C1}Processor Type${CN} you wish to run on:"
+      echo "   ${C2}sky  (Skylake)${CN}"
+      echo "   ${C2}cas  (Cascade Lake)${CN} (default)"
+      echo " "
+      set MODEL = `echo $<`
+      set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"`
+      if ( .$MODEL == .) then
+         set MODEL = 'cas'
+      endif
+
+      if( $MODEL != 'sky'  & \
+          $MODEL != 'cas' ) goto ASKPROC
+
+      if ($MODEL == 'sky') then
+         set NCPUS_PER_NODE = 40
+      else if ($MODEL == 'cas') then
+         # NCCS currently recommends that users do not run with
+         # 48 cores per node on SCU16 due to OS issues and
+         # recommends that CPU-intensive works run with 46 or less
+         # cores. As 45 is a multiple of 3, it's the best value
+         # that doesn't waste too much
+         #set NCPUS_PER_NODE = 48
+         set NCPUS_PER_NODE = 45
+      endif
    endif
 
 else if ( $SITE == 'NAS' ) then
@@ -708,7 +730,7 @@ EOF
 # The below settings seem to be recommended for hybrid
 # systems using MVAPICH2 but could change
 
-else if( $MPI == mvapich2 ) then
+else if( $MPI == mvapich ) then
 
 cat > $EXPDIR/SETENV.commands << EOF
    setenv MV2_ENABLE_AFFINITY     0
@@ -778,6 +800,17 @@ EOF
 
 endif # if NOT Singularity
 
+# Testing on SLES15 showed that the mlx provider did not seem
+# to work at scale. So we move to use the verbs provider. Note:
+# still seems to have issues at c720
+if ("$BUILT_ON_SLES15" == "TRUE") then
+cat >> $EXPDIR/SETENV.commands << EOF
+setenv I_MPI_OFI_PROVIDER verbs
+setenv I_MPI_COLL_EXTERNAL 0
+EOF
+
+endif # if SLES15
+
 endif # if NCCS
 
 endif # if mpi