runmpi

#!/bin/sh
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
ulimit -s unlimited
[ -z "$nprocs" ] && echo "Need to set nprocs" && exit 1;
[ -z "$PGM" ] && echo "Need to set PGM" && exit 1;
totcores=`expr $nprocs \* $OMP_NUM_THREADS`
if [ $machine == 'noaacloud' ]; then
  sruncmd="srun --mpi=pmi2 --export=ALL"
  python=/usr/bin/python
else
  sruncmd="srun"
  python=`which python`
fi
if [ $machine == 'gaeac5' ]; then
  export FI_VERBS_PREFER_XRC=0 
fi
totnodes=`$python -c "import math; print(int(math.ceil(float(${totcores})/${corespernode})))"`
count=`$python -c "import math; print(int(math.floor(float(${corespernode})/${mpitaskspernode})))"` 
mpitaskspernode=`$python -c "import math; print(int(math.ceil(float(${nprocs})/${totnodes})))"`
# -c: cpus per mpi task (number of threads per mpi task)
# -n: total number of mpi tasks
# -N: number of nodes to run on
# --ntasks-per-node:  mpi tasks on each node
echo "running $sruncmd -N $totnodes -n $nprocs -c $count --ntasks-per-node=$mpitaskspernode  --exclusive --cpu-bind=cores --verbose $PGM"
eval $sruncmd -N $totnodes -n $nprocs -c $count --ntasks-per-node=$mpitaskspernode --exclusive --cpu-bind=cores --verbose $PGM
rc=$?
echo "exiting runmpi..."
exit $rc