fsl_sub

#!/bin/sh

# Jul 2013 - set swarm to autobundle (Susan)

# Mar 2012: FEAT was sending non-numeric job IDs as dependencies, e.g. -j featModelID which gets
#   translated by this script to 'swarm ... -W=depend:afterok:featModelID' which fails.
#   Modified this script to test for numeric value for -j flag. If non-numeric, the flag is ignored. (Susan)
#
# modified by Adam Thomas, adamt@nih.gov. Aug 2008 to use swarm instead of SGE
# further modified by Susan, Sep 2009 


# Copyright (C) 2007 University of Oxford
# Authors: Dave Flitney & Stephen Smith

#   Part of FSL - FMRIB's Software Library
#   http://www.fmrib.ox.ac.uk/fsl
#   fsl@fmrib.ox.ac.uk
#   
#   Developed at FMRIB (Oxford Centre for Functional Magnetic Resonance
#   Imaging of the Brain), Department of Clinical Neurology, Oxford
#   University, Oxford, UK
#   
#   
#   LICENCE
#   
#   FMRIB Software Library, Release 4.0 (c) 2007, The University of
#   Oxford (the "Software")
#   
#   The Software remains the property of the University of Oxford ("the
#   University").
#   
#   The Software is distributed "AS IS" under this Licence solely for
#   non-commercial use in the hope that it will be useful, but in order
#   that the University as a charitable foundation protects its assets for
#   the benefit of its educational and research purposes, the University
#   makes clear that no condition is made or to be implied, nor is any
#   warranty given or to be implied, as to the accuracy of the Software,
#   or that it will be suitable for any particular purpose or for use
#   under any specific conditions. Furthermore, the University disclaims
#   all responsibility for the use which is made of the Software. It
#   further disclaims any liability for the outcomes arising from using
#   the Software.
#   
#   The Licensee agrees to indemnify the University and hold the
#   University harmless from and against any and all claims, damages and
#   liabilities asserted by third parties (including claims for
#   negligence) which arise directly or indirectly from the use of the
#   Software or the sale of any products based on the Software.
#   
#   No part of the Software may be reproduced, modified, transmitted or
#   transferred in any form or by any means, electronic or mechanical,
#   without the express permission of the University. The permission of
#   the University is not required if the said reproduction, modification,
#   transmission or transference is done without financial return, the
#   conditions of this Licence are imposed upon the receiver of the
#   product, and all original and amended source code is included in any
#   transmitted product. You may be held legally responsible for any
#   copyright infringement that is caused or encouraged by your failure to
#   abide by these terms and conditions.
#   
#   You are not permitted under this Licence to use this Software
#   commercially. Use for which any financial return is received shall be
#   defined as commercial use, and includes (1) integration of all or part
#   of the source code or the Software into a product for sale or license
#   by or on behalf of Licensee to third parties or (2) use of the
#   Software or any derivative of it for research with the final aim of
#   developing software products for sale or license to a third party or
#   (3) use of the Software or any derivative of it for research with the
#   final aim of developing non-software products for sale or license to a
#   third party, or (4) use of the Software to provide any service to an
#   external organisation for which payment is received. If you are
#   interested in using the Software commercially, please contact Isis
#   Innovation Limited ("Isis"), the technology transfer company of the
#   University, to negotiate a licence. Contact details are:
#   innovation@isis.ox.ac.uk quoting reference DE/1112.

export FSLSUBVERBOSE=1




###########################################################################
# Edit this file in order to setup FSL to use your local compute
# cluster.
###########################################################################


###########################################################################
# The following section determines what to do when fsl_sub is called
# by an FSL program. If it finds a local cluster if will pass the
# commands onto the cluster. Otherwise it will run the commands
# itself. There are two values for the METHOD variable, "SGE" and
# "NONE". You should setup the tests to look for whether the calling
# computer can see your cluster setup scripts, and run them (if that's
# what you want, i.e. if you haven't already run them in the user's
# login scripts). Note that these tests look for the environment
# variable SGE_ROOT, which a user can unset if they don't want the
# cluster to be used.
###########################################################################

METHOD=PBS


###########################################################################
# The following auto-decides what cluster queue to use. The calling
# FSL program will probably use the -T option when calling fsl_sub,
# which tells fsl_sub how long (in minutes) the process is expected to
# take (in the case of the -t option, how long each line in the
# supplied file is expected to take). You need to setup the following
# list to map ranges of timings into your cluster queues - it doesn't
# matter how many you setup, that's up to you.
###########################################################################

map_qname ()
{
    if [ $1 -le 20 ] ; then
	queue=veryshort.q
    elif [ $1 -le 120 ] ; then
	queue=short.q
    elif [ $1 -le 1440 ] ; then
	queue=long.q
    else
	queue=verylong.q
    fi
    #echo "Estimated time was $1 mins: queue name is $queue"
}


###########################################################################
# Don't change the following (but keep scrolling down!)
###########################################################################

POSIXLY_CORRECT=1
export POSIXLY_CORRECT
command=`basename $0`

usage ()
{
  cat <<EOF

$command V1.0beta - wrapper for job control system such as PBS

Usage: fsl_sub [options] <command>

$command gzip *.img *.hdr
$command -q short.q gzip *.img *.hdr
$command -a darwin regscript rawdata outputdir ...

  -T <minutes>          Estimated job length in minutes, used to auto-set queue name **NOTUSED
  -a <node type>        e.g. 'm4096' (4 GB memory) **NOTUSED
  -M <email-address>    Who to email, default = `whoami`@fmrib.ox.ac.uk  **NOTUSED
  -j <jid>              Place a hold on this task until job jid has completed
  -t <filename>         Specify a task file of commands to execute in parallel
  -N <jobname>          Specify jobname as it will appear on queue
  -l <logdirname>       Where to output logfiles
  -m <mailoptions>      Change the PBS mail options, see qsub for details
  -F                    Use flags embedded in scripts to set PBS queuing options
  -s <pename>,<threads> Submit a multi-threaded task - requires a PE (<pename>) to be
                        configured for the requested queues.
                        <threads> specifies the number of threads to run
  -z <output>           If <output> image or file already exists, do nothing and exit
  -v                    Verbose mode.

EOF

  exit 1
}

nargs=$#
if [ $nargs -eq 0 ] ; then
  usage
fi

set -- `getopt T:q:a:p:M:j:t:z:N:Fvm:l:s: $*`
result=$?
if [ $result != 0 ] ; then
  echo "What? Your arguments make no sense!"
fi

if [ $nargs -eq 0 ] || [ $result != 0 ] ; then
  usage
fi


###########################################################################
# The following sets up the default queue name, which you may want to
# change. It also sets up the basic emailing control.
###########################################################################

queue=long.q
mailto=`whoami`@biobos.nih.gov
#MailOpts="abe"
MailOpts="a"


###########################################################################
# In the following, you might want to change the behaviour of some
# flags so that they prepare the right arguments for the actual
# cluster queue submission program, in our case "qsub".
#
# -a sets is the cluster submission flag for controlling the required
# hardware architecture (normally not set by the calling program)
#
# -p set the priority of the job - ignore this if your cluster
# environment doesn't have priority control in this way.
#
# -j tells the cluster not to start this job until cluster job ID $jid
# has completed. You will need this feature.
#
# -t will pass on to the cluster software the name of a text file
# containing a set of commands to run in parallel; one command per
# line.
#
# -N option determines what the command will be called when you list
# running processes.
#
# -l tells the cluster what to call the standard output and standard
# -error logfiles for the submitted program.
###########################################################################

if [ -z $FSLSUBVERBOSE ] ; then
    verbose=0
else
    verbose=$FSLSUBVERBOSE;
    echo "Input args=$@" >&2
    echo "" >&2
fi

scriptmode=0

while [ $1 != -- ] ; do
  case $1 in
    -z)
      if [ -e $2 -o `${FSLDIR}/bin/imtest $2` = 1 ] ; then
        exit 0
      fi
      shift;;
    -T)
      map_qname $2
      shift;;
    -a)
      pbs_arch=":$2"
      shift;;
    -M)
      mailto=$2
      shift;;
    -j)
      jid=$2
      jid=${jid//biobos,/biobos:}
      if echo $jid | grep "^[0-9]" > /dev/null
          then
          pbs_hold="-W depend=afterok:$jid"
          else
          echo "Warning: -j $jid is not a jobID. Ignoring the dependency" >&2
          fi
      shift;;
    -t)
      taskfile=$2
      tasks=`wc -l $taskfile | awk '{print $1}'`
      pbs_tasks="-t 1-$tasks"
      shift;;
    -N)
      JobName=$2;
      shift;;
# bedpost adds some SGE mail flags which don't work in PBS. Safer to ignore mail options
#   when fsl_sub is called.
#    -m)
#      MailOpts=$2;
#      shift;;
    -l)
      #LogOpts="-o $2 -e $2";
      LogDir="${2}/";
      mkdir -p $2;
      shift;;
    -F)
      scriptmode=1;
      ;;
    -v)
      verbose=1
      ;;
    -s)
      pe_string=$2;
      peName=`echo $pe_string | cut -d',' -f 1`
      peThreads=`echo $pe_string | cut -d',' -f 2`
      peThreads="-t $peThreads"
      shift;;
  esac
  shift  # next flag
done
shift

###########################################################################
# Don't change the following (but keep scrolling down!)
###########################################################################

if [ "x$JobName" = x ] ; then 
    if [ "x$taskfile" != x ] ; then
	JobName=`basename $taskfile`
    else
	JobName=`basename $1`
	# Names must be all alphabetic characters. This is only a partial fix
	JobName=${JobName//[-_.0-9]/}
	# And they can only be 15 chars long.
	JobName=${JobName:0:15}
    fi
fi

#date >> /tmp/agt_debug
#echo jobname $JobName >> /tmp/agt_debug
#echo taskfile $taskfile >> /tmp/agt_debug
#echo arg1 $1 >> /tmp/agt_debug


if [ "x$tasks" != x ] && [ ! -f "$taskfile" ] ; then
    echo $taskfile: invalid input!
    echo Should be a text file listing all the commands to run!
    exit -1
fi

if [ "x$tasks" != "x" ] && [ "x$@" != "x" ] ; then
    echo $@
    echo Spurious input after parsing command line!
    exit -1
fi

case $METHOD in

###########################################################################
# The following is the main call to the cluster, using the "qsub" PBS
# program. If $tasks has not been set then qsub is running a single
# command, otherwise qsub is processing a text file of parallel
# commands.
###########################################################################

    PBS)
	if [ "x$tasks" = "x" ] ; then
		pid=$$
		echo $@ "; rm `pwd`/cmd.$pid" > cmd.$pid
		chmod +x cmd.$pid
		pbs_command="swarm -R gpfs --autobundle -g 4 -f cmd.$pid -V $LogOpts $pbs_hold $peThreads -j oe"
#	    if [ $scriptmode -ne 1 ] ; then
#		pbs_command="qsub -V -cwd -shell n -b y -r y -q $queue -M $mailto -N $JobName -m $MailOpts $LogOpts $pbs_arch $pbs_hold"
#	    else
#		pbs_command="qsub $LogOpts $pbs_arch $pbs_hold"
#	    fi
	    if [ $verbose -eq 1 ] ; then 
		echo pbs_command: $pbs_command >&2
		echo executing: $@ >&2
	    fi
#	    exec $pbs_command $@ | awk '{print $3}'
#	    date >> /tmp/agt_debug
#	    echo $pbs_command >> /tmp/agt_debug
	    exec $pbs_command
	else
	    #pbs_command="qsub -V -cwd -M $mailto -N $JobName -m $MailOpts $LogOpts $pbs_hold $pbs_tasks"
	   pbs_command="swarm -R gpfs --autobundle -g 4 -f $taskfile $pbs_hold -V $peThreads -j oe --singleout -v FSLOUTPUTTYPE=$FSLOUTPUTTYPE"; 
	   if [ $verbose -eq 1 ] ; then 
		echo pbs_command: $pbs_command >&2
		echo control file: $taskfile >&2
	    fi
            ID=`swarm -R gpfs --autobundle -g 4 -f $taskfile $pbs_hold -V $peThreads --singleout -v FSLOUTPUTTYPE=$FSLOUTPUTTYPE`
            ID=${ID//biobos?/biobos:} 
            echo $ID

#	    exec $pbs_command <<EOF | awk '{print $3}' | awk -F. '{print $1}'
##!/bin/sh
#
##$ -S /bin/sh
#
#command=\`sed -n -e "\${PBS_TASK_ID}p" $taskfile\`
#
#exec /bin/sh -c "\$command"
#EOF
	fi
	;;

###########################################################################
# Don't change the following - this runs the commands directly if a
# cluster is not being used.
###########################################################################

    NONE)
	if [ "x$tasks" = "x" ] ; then
	    if [ $verbose -eq 1 ] ; then 
		echo executing: $@ >&2
	    fi

	    /bin/sh <<EOF1 > ${LogDir}${JobName}.o$$ 2> ${LogDir}${JobName}.e$$
$@
EOF1
	else
	    if [ $verbose -eq 1 ] ; then 
		echo "Running commands in: $taskfile" >&2
	    fi

	    n=1
	    while [ $n -le $tasks ] ; do
		line=`sed -n -e ''${n}'p' $taskfile`
		if [ $verbose -eq 1 ] ; then 
		    echo executing: $line >&2
		fi
		/bin/sh <<EOF2 > ${LogDir}${JobName}.o$$.$n 2> ${LogDir}${JobName}.e$$.$n
$line
EOF2
		n=`expr $n + 1`
	    done
	fi	
	echo $$
	;;

esac

###########################################################################
# Done.
###########################################################################