change to apps list to use URLs (docker://, shub://, file://), and sh…

…ort-hand names to specify the app instead of app+version

README.md

@@ -31,14 +31,23 @@ Compute Canada does not allow any GUI (X) applications to be run, even in intera
 
 ## BIDS Apps:
 
-Running bidsBatch lists the usage and displays bids-apps that are deployed on the system (contents of bids-apps.tsv).  For default app options, please also see the bids-apps.tsv file.  Note: when you run bidsBatch to process a dataset, a ```./jobs/``` folder is created in the current working directory.
+The bidsBatch wrapper uses SLURM to parallelize over *subjects*, and by default will run a job for every subject in your participants.tsv file. 
+bidsBatch uses SLURM Arrays, so it groups all the different jobs under a single job ID, with each array job indicated as <jobid>_<index>.
+
+Running bidsBatch lists the usage and displays bids-apps that are deployed on the system (contents of bids-apps.tsv).  For default app options, please also see the bids-apps.tsv file.  Note: when you run bidsBatch to process a dataset, a ```jobs/``` folder is created in the output directory you specify.
+
 
 ### Example: Running fmriprep on your bids dataset
 ```
-mkdir fmriprep_work
-cd fmriprep_work
-bidsBatch fmriprep 1.0.4 ~/my-bids-dataset ~/my-bids-dataset/derivatives/fmriprep-v1.0.4 participant 
+bidsBatch fmriprep_1.0.4 ~/my-bids-dataset ~/my-bids-dataset/derivatives/fmriprep-v1.0.4 participant 
+```
+To get usage/options for a particular bids app, leave out the bids-app required arguments, e.g.:
+
 ```
+bidsBatch fmriprep_1.0.4 
+```
+
+
 
 ## Wrappers:
 

bids-apps.tsv

@@ -1,15 +1,10 @@
-app_name	tag	repo	base_options	job_template
-fmriprep	1.0.4	poldracklab	--nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor
-fmriprep	1.0.0	poldracklab	--nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor
-fmriprep	1.0.0-rc8	poldracklab	--nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID}
-mriqc	0.10.0	poldracklab	--no-sub --n_procs 8 --mem_gb 32000 -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID}
-niak	latest	bids	--n_thread 8
-brainiak-srm	latest	bids
-tracula	v6.0.0-4	bids
-prepdwi	0.0.3b	khanlab		2core8gb16h
-magetbrain	latest	bids
-spm	latest	bids
-ndmg	v0.0.50	bids
-cpac	v1.0.2_disable_log_2	bids	
-mrtrix3_connectome	0.2.2	bids
-qsm-sstv	v0.0.1	khanlab	--SS_TV_lagrange_parameter 0.4 --mag_match_pattern *part-mag_echo* --phase_match_pattern *part-phase_echo* --keep_unnecessary_outputs
+name	url	base_options	job_template
+fmriprep_1.0.6	docker://poldracklab/fmriprep:1.0.6	--nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor
+fmriprep_1.0.4	docker://poldracklab/fmriprep:1.0.4	--nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor
+mriqc_0.10.1	docker://poldracklab/mriqc:0.10.1	--no-sub --n_procs 8 --mem_gb 32000 -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID}
+mriqc_0.10.0	docker://poldracklab/mriqc:0.10.0	--no-sub --n_procs 8 --mem_gb 32000 -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID}
+prepdwi_0.0.4b	shub://khanlab/prepdwi:0.0.4b	--no-bedpost	2core8gb16h
+prepdwi_0.0.4b_bedpost	shub://khanlab/prepdwi:0.0.4b	--n_cpus 32	Fat
+prepdwi_0.0.3e	file:///project/6007967/akhanf/singularity/bids-apps/khanlab_prepdwi_0.0.3e.img	--no-bedpost	2core8gb16h
+qsm_sstv_v0.0.1	shub://khanlab/qsm_sstv:v0.0.1	--SS_TV_lagrange_parameter 0.4 --mag_match_pattern *part-mag_echo* --phase_match_pattern *part-phase_echo* --keep_unnecessary_outputs
+mp2rage_correction_v0.0.1	shub://khanlab/mp2rage_correction:v0.0.1		ShortSkinny

bidsBatch

@@ -8,10 +8,10 @@ function usage {
    echo "  Loops through an input subject_list_txt (txt with subj id on each line)"
    echo "  Can be used to run scripts that generally take command-line parameters as:"
    echo ""
-   echo "Usage: $cmdname <optional flags> <app_name> <version> <bids_dir> <output_dir> <participant/group> <app options>"
+   echo "Usage: $cmdname <optional flags> <app_name> <bids_dir> <output_dir> <participant/group> <app options>"
    echo ""
    echo "Available apps:"
-	cat $app_list | awk -F '\t' '{print $1 "\t"  $2}'
+	cat $app_list | awk -F '\t' '{print $1 "\t"  $2}' | column -t
    echo ""
    echo "optional flags:"
    echo ""
@@ -33,6 +33,87 @@ function usage {
  }
 
 
+function getContainer {
+
+app_name=$1
+bids_app_dir=$SINGULARITY_DIR/bids-apps
+
+#get matching line for app_name
+linenum=`awk -F '\t' '{print $1}' $app_list  | grep -n $app_name`
+linenum=${linenum%%:*}
+
+
+if [ ! -n "$linenum" ]
+then
+	echo "${app_name} does not exist in bids-app list!"  >&2 
+	exit 1
+fi
+
+url=`cat $app_list | head -n $linenum | tail -n 1 |  awk -F '\t' '{print $2}'`
+base_opts=`cat $app_list | head -n $linenum | tail -n 1 |  awk -F '\t' '{print $3}'`
+app_job_template=`cat $app_list | head -n $linenum | tail -n 1 |  awk -F '\t' '{print $4}'`
+
+hub=${url%%:*}
+if [ "$hub" = "docker" ]
+then
+  #location for deployed docker images
+  repo=${url#*://}
+  repo=${repo%%/*}
+
+  app=${url##*/}
+  app=${app%:*}
+
+  tag=${url##*:}
+
+  singularity_image=$bids_app_dir/${repo}_${app}_${tag}.img
+if [ ! -e $singularity_image ]
+ then 
+	 echo "Singularity image: $singularity_image  not found," >&2 
+	echo "   run deploy_bids-apps first on your local machine" >&2 
+	 exit 1
+ fi
+
+
+fi
+
+if [ "$hub" = "shub" ]
+then
+ #use shub-cache to download if not exist, and return path to image
+ singularity_image=`shub-cache $url`
+ if [ ! "$?" = 0  -o ! -n "$singularity_image" -o ! -e "$singularity_image"  ]
+ then
+   echo "Failed to pull $url, exiting" >&2
+   exit 1
+ fi
+
+
+fi
+
+if [ "$hub" = "file" ]
+then
+ singularity_image=${url##file://}
+
+ if [  ! -e "$singularity_image"  ]
+ then
+   echo "File $singularity_image does not exist,  exiting" >&2
+   exit 1
+ fi
+
+
+fi
+
+
+if [ "$override_template" = "0" -a -n "$app_job_template" ]
+then
+	job_template=$app_job_template 
+	echo "Using job-template specified in bids-apps.tsv, $app_job_template" >&2
+fi
+
+
+
+
+}
+
 
 execpath=`dirname $0`
 execpath=`realpath $execpath`
@@ -83,57 +164,60 @@ done
 
 shift $((OPTIND-1))
 
-if [ "$#" -lt 5 ]
+if [ "$#" -lt 1 ]
 then
 	usage
 	exit 1
 fi
 
-app_name=$1
-tag=$2
-bids_dir=`realpath $3`
-out_dir=$4
-level=$5
 
+app_name=$1
 
+shift 1
 
-mkdir -p $out_dir
-out_dir=`realpath $out_dir`
-
-shift 5
-options=$@
+getContainer $app_name
 
 
 
-bids_app_dir=$SINGULARITY_DIR/bids-apps
-repo=`grep "${app_name}	${tag}	" $app_list | awk -F '\t' '{print $3}'`
-
-if [ ! -n "$repo"  ]
+if [ "$#" -lt 3 ]
 then
-	echo "${app_name} ${tag} does not exist in bids-app list!"  >&2 
+	usage
+	echo "Usage for bids app:" >&2
+	if [ -e $singularity_image ]
+	then
+		singularity run $singularity_image
+	fi
+
 	exit 1
 fi
 
-base_opts=`grep "${app_name}	${tag}	" $app_list | awk -F '\t' '{print $4}'`
-app_job_template=`grep "${app_name}	${tag}	" $app_list | awk -F '\t' '{print $5}'`
+bids_dir=`realpath $1`
+out_dir=$2
+level=$3
 
-if [ "$override_template" = "0" -a -n "$app_job_template" ]
+#check validity of participant 
+level_part=${level:0:5}
+if [   "$level_part" != "group" -a  "$level_part" != "parti" ]
 then
-	job_template=$app_job_template 
-	echo "Using job-template specified in bids-apps.tsv, $app_job_template" >&2
+	echo "Required argument 3 to bids apps must be analysis level (participant/group)" >&2
+	exit 1
 fi
 
+mkdir -p $out_dir
+out_dir=`realpath $out_dir`
 
+shift 3
+options=$@
 
-singularity_image=$bids_app_dir/${repo}_${app_name}_${tag}.img
 
+#check if participant_label is used as an option 
+#and quit if it is found
 
-if [ ! -e $singularity_image ]
- then 
-	 echo "Singularity image: $singularity_image  not found," >&2 
-	echo "   run deploy_bids-apps first on your local machine" >&2 
-	 exit 1
- fi
+if [[ $options = *--participant_label* ]]
+then
+  echo "ERROR: --participant_label should not be used with bidsBatch, use: bidsBatch -s <participant_label> instead" >&2
+ exit 1
+fi  
 
 
 
@@ -205,7 +289,7 @@ cp $execpath/job-templates/${job_template}.job $job
 
 echo "#SBATCH --job-name=$app_name" >> $job
 
-
+#FIX THIS LATER TO ALLOW FOR  participant1 etc.., group1, group2 ...
 if [ "$level" == "participant" ]
 then
 	#submit with array, using subj and subjlist, indexed by line number (1-N)
@@ -216,12 +300,12 @@ then
 	#strip off sub- from sub-LABEL
 	echo "subj=\${subj##sub-}" >> $job
 	echo "echo SINGULARITY_IMAGE: $singularity_image" >> $job
-	echo "echo CMD: --participant_label \$subj $base_opts $options $bids_dir $out_dir $level" >>$job
+	echo "echo CMD: $bids_dir $out_dir $level --participant_label \$subj $base_opts $options" >>$job
 	echo "echo START_TIME: \`date\`" >>$job
 	echo "SCRATCH_DIR=/scratch/${USER}/\${SLURM_ARRAY_JOB_ID}_\${SLURM_ARRAY_TASK_ID}" >> $job
 	echo "mkdir -p \$SCRATCH_DIR" >> $job
 	echo "cd `pwd`" >> $job
-	echo singularity run $SINGULARITY_OPTS $singularity_image --participant_label \$subj $base_opts $options $bids_dir $out_dir $level  >> $job
+	echo singularity run $SINGULARITY_OPTS $singularity_image $bids_dir $out_dir $level  --participant_label \$subj $base_opts $options >> $job
 	echo "RETURNVAL=\$?" >> $job
 	echo "rm -rf \$SCRATCH_DIR" >> $job
 	echo "echo RETURNVAL=\$RETURNVAL" >>$job
@@ -233,12 +317,12 @@ else
 	#submit without arrays, for group level
 	echo "#SBATCH --output=$job_dir/${app_name}.$level.%A.out" >> $job
 	echo "echo SINGULARITY_IMAGE: $singularity_image" >> $job
-	echo "echo CMD: $base_opts $options $bids_dir $out_dir $level" >>$job
+	echo "echo CMD:  $bids_dir $out_dir $level $base_opts $options" >>$job
 	echo "echo START_TIME: \`date\`" >>$job
 	echo "export SCRATCH_DIR=/scratch/${USER}/\${SLURM_JOB_ID}" >> $job
 	echo "mkdir -p \$SCRATCH_DIR" >> $job
 	echo "cd `pwd`" >> $job
-	echo singularity run $SINGULARITY_OPTS $singularity_image $base_opts $options $bids_dir $out_dir $level  >> $job
+	echo singularity run $SINGULARITY_OPTS $singularity_image $bids_dir $out_dir $level $base_opts $options >> $job
 	echo "RETURNVAL=\$?" >> $job
 	echo "rm -rf \$SCRATCH_DIR" >> $job
 	echo "echo RETURNVAL=\$RETURNVAL" >>$job

deploy_bids-apps

@@ -11,13 +11,24 @@ Nlines=`cat $app_list | wc -l`
 
 for i in `seq 2 $Nlines`
 do
- app=`cat $app_list | head -n $i | tail -n 1 | awk  -F '\t' '{print $1}'`
- tag=`cat $app_list | head -n $i | tail -n 1 | awk  -F '\t' '{print $2}'`
- repo=`cat $app_list | head -n $i | tail -n 1 | awk  -F '\t' '{print $3}'`
- base_opts=`cat $app_list | head -n $i | tail -n 1 | awk  -F '\t' '{print $4}'`
- docker_name=${repo}/${app}:${tag}
- out_name=${repo}_${app}_${tag}.img
- out_img=$remote_dir/$out_name
+ appname=`cat $app_list | head -n $i | tail -n 1 | awk  -F '\t' '{print $1}'`
+ url=`cat $app_list | head -n $i | tail -n 1 | awk  -F '\t' '{print $2}'`
+
+
+ hub=${url%%:*}
+ if [ "$hub" = "docker" ]
+ then
+  docker_name=${url#docker://}
+
+  repo=${url#*://}
+  repo=${repo%%/*}
+
+  app=${url##*/}
+  app=${app%:*}
+
+  tag=${url##*:}
+
+  out_img=$remote_dir/${repo}_${app}_${tag}.img
 
  if [ ! -e $out_img ]
  then
@@ -37,4 +48,5 @@ do
   echo Skipping $out_img, already exists ...
  fi
 
+ fi
 done