forked from StormSurgeLive/asgs
-
Notifications
You must be signed in to change notification settings - Fork 0
/
asgs_main.sh
executable file
·1936 lines (1933 loc) · 90.3 KB
/
asgs_main.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#----------------------------------------------------------------
#
# asgs_main.sh: This is the main driver script for the ADCIRC Surge Guidance
# System (ASGS). It performs configuration tasks via config.sh, then enters a
# loop which is executed once per advisory cycle.
#
#----------------------------------------------------------------
# Copyright(C) 2006--2016 Jason Fleming
# Copyright(C) 2006, 2007 Brett Estrade
#
# This file is part of the ADCIRC Surge Guidance System (ASGS).
#
# The ASGS is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ASGS is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with the ASGS. If not, see <http://www.gnu.org/licenses/>.
#----------------------------------------------------------------
#
#####################################################################
# B E G I N F U N C T I O N S
#####################################################################
#
echoHelp()
{ clear
echo "@@@ Help @@@"
echo "Usage:"
echo " bash %$0 [-s /full/path/to/statefile] [-c /fullpath/of/asgs_config.sh] -e environment"
echo
echo "Options:"
echo "-c : set location of configuration file"
echo "-e (environment): set the computer that the ASGS is running on"
echo "-s : start from a previous statefile (used when started by cron)"
echo "-h : show help"
exit;
}
#
# subroutine to check for the existence of required files that have
# been specified in config.sh
checkFileExistence()
{ FPATH=$1
FTYPE=$2
FNAME=$3
if [[ -z $FNAME ]]; then
fatal "The $FTYPE was not specified in the configuration file. When it is specified, the ASGS will look for it in the path ${FPATH}."
fi
if [ $FNAME ]; then
if [ -e "${FPATH}/${FNAME}" ]; then
logMessage "The $FTYPE '${FPATH}/${FNAME}' was found."
else
fatal "The $FTYPE '${FPATH}/${FNAME}' does not exist."
fi
fi
}
#
# compare the modification times of the input files with the archive of
# subdomain files to avoid using a stale archive
checkArchiveFreshness()
{ PREPPEDARCHIVE=$1
HINDCASTARCHIVE=$2
GRIDFILE=$3
CONTROLTEMPLATE=$4
ELEVSTATIONS=$5
VELSTATIONS=$6
METSTATIONS=$7
NAFILE=$8
INPUTDIR=$9
logMessage "Checking to see if the archive of preprocessed subdomain files is up to date."
for archiveFile in $PREPPEDARCHIVE $HINDCASTARCHIVE; do
if [ ! -e $INPUTDIR/$archiveFile ]; then
logMessage "The subdomain archive file $INPUTDIR/$archiveFile does not exist."
continue
fi
for inputFile in $GRIDFILE $CONTROLTEMPLATE $ELEVSTATIONS $VELSTATIONS $METSTATIONS $NAFILE; do
if [ ! -e $INPUTDIR/$inputFile ]; then
warn "The input file $INPUTDIR/$inputFile does not exist."
continue
fi
# see if the archiveFile is older than inputFile
if [ $INPUTDIR/$archiveFile -ot $INPUTDIR/$inputFile ]; then
logMessage "A change in the input files has been detected. The archive file $archiveFile is older than the last modification time of the input file ${inputFile}. The archive file is therefore stale and will be deleted. A fresh one will automatically be created the next time adcprep is run."
rm $INPUTDIR/$archiveFile 2>> $SYSLOG
break
fi
done
done
}
#
# subroutine to check for the existence of required directories
# that have been specified in config.sh
checkDirExistence()
{ DIR=$1
TYPE=$2
if [[ -z $DIR ]]; then
fatal "The $TYPE was not specified in the configuration file."
fi
if [[ -e $DIR ]]; then
logMessage "The $TYPE '$DIR' was found."
else
fatal "The $TYPE '$DIR' does not exist."
fi
}
#
# subroutine to check for the existence and nonzero length of the
# hotstart file
# the subroutine assumes that the hotstart file is named fort.$LUN or
# fort.$LUN.nc depending on the format and expects it to be provided with
# the full path if it is not in the current directory.
checkHotstart()
{
FROMDIR=$1
HOTSTARTFORMAT=$2
LUN=$3
#
HOTSTARTFILE=''
# set name and specific file location based on format (netcdf or binary)
if [[ $HOTSTARTFORMAT = netcdf ]]; then
HOTSTARTFILE=$FROMDIR/fort.$LUN.nc
else
HOTSTARTFILE=$FROMDIR/PE0000/fort.$LUN
fi
# check for existence of hotstart file
if [ ! -e $HOTSTARTFILE ]; then
fatal "The hotstart file '$HOTSTARTFILE' was not found. The preceding simulation run must have failed to produce it."
# if it exists, check size to be sure its nonzero
else
hotstartSize=`stat -c %s $HOTSTARTFILE`
if [ $hotstartSize == "0" ]; then
fatal "The hotstart file '$HOTSTARTFILE' is of zero length. The preceding simulation run must have failed to produce it properly."
else
logMessage "The hotstart file '$HOTSTARTFILE' was found and it contains $hotstartSize bytes."
# check time in hotstart file to be sure it can be found and that
# it is nonzero
# jgf20170131: hstime reports errors to stderr so we must capture
# that with backticks and tee to the log file
HSTIME=''
if [[ $HOTSTARTFORMAT = netcdf ]]; then
HSTIME=`$ADCIRCDIR/hstime -f $HOTSTARTFILE -n 2>&1 | tee --append ${SYSLOG}`
else
HSTIME=`$ADCIRCDIR/hstime -f $HOTSTARTFILE 2>&1 | tee --append ${SYSLOG}`
fi
failureOccurred=$?
errorOccurred=`expr index "$HSTIME" ERROR`
if [[ $failureOccurred != 0 || $errorOccurred != 0 ]]; then
fatal "The hstime utility could not read the ADCIRC time from the file '$HOTSTARTFILE'. The output from hstime was as follows: '$HSTIME'."
else
if float_cond '$HSTIME == 0.0'; then
fatal "The time in the hotstart file '$HOTSTARTFILE' is zero. The preceding simulation run must have failed to produce a proper hotstart file."
fi
fi
fi
fi
}
#
# Evaluate a floating point number conditional expression.
# From http://www.linuxjournal.com/content/floating-point-math-bash
function float_cond()
{
local cond=0
if [[ $# -gt 0 ]]; then
cond=$(echo "$*" | bc -q 2>/dev/null)
if [[ -z "$cond" ]]; then cond=0; fi
if [[ "$cond" != 0 && "$cond" != 1 ]]; then cond=0; fi
fi
local stat=$((cond == 0))
return $stat
}
#
# Retrieve and build ADCIRC(+SWAN) executables. This function will set
# the value of ADCIRCDIR if ADCIRCBUILD = "dynamic".
get_adcirc()
{
ADCIRCDIR=$1 # this value may be changed in this function
DEBUG=$2
SWAN=$3
NETCDF=$4
NETCDF4=$5
NETCDF4_COMPRESSION=$6
XDMF=$7
SOURCEURL=$8
AUTOUPDATE=$9
EXEBASEPATH=${10}
SCRIPTDIR=${11}
SWANMACROSINC=${12}
ADCOPTIONS="${13}"
SYSLOG=${14}
#
# If the path to the ADCIRC executables is hard coded, just verify
# their existence and return.
if [[ $ADCIRCBUILD = static ]]; then
for executable in adcirc padcirc padcswan adcprep hstime aswip; do
if [[ ! -e $ADCIRCDIR/$executable ]]; then
warn "Could not find the executable file $ADCIRCDIR/${executable}."
return 1
fi
done
# leave the value of ADCIRCDIR as-is
logMessage "All ADCIRC(+SWAN) executable files were found successfully."
return 0
fi
#
# Set the name of the properties file that describes the executables that
# we're about to generate.
PROPERTIES=executables.properties
#
logMessage "Checking for suitable ADCIRC(+SWAN) executables."
#
# Check to see if we already have executables in a directory that
# matches the specification.
#
# Start by generating the path that is specified by this combination of
# parameters. Assume the SOURCEURL is an http URL for an svn repository;
# extract the end of the path for use in naming the directory where the
# executables will be compiled.
EXEPATH=`basename $SOURCEURL`
#
# Add the first letter of each of the arguments to the path name
EXEPATH="${EXEPATH}_D${DEBUG:0:1}S${SWAN:0:1}N${NETCDF:0:1}N4${NETCDF4:0:1}N4C${NETCDF4_COMPRESSION:0:1}X${XDMF:0:1}"
#
# Prepend the base executables path to the path we've constructed.
EXEPATH=$EXEBASEPATH/$EXEPATH
#
# Check for existence of executables.
EXEFOUND=t
for executable in adcirc padcirc padcswan adcprep hstime aswip; do
if [[ ! -e $EXEPATH/work/$executable ]]; then
EXEFOUND=f
fi
done
#
# If we found all the executables, and we aren't supposed to try to
# update and recompile them, then we're done.
if [[ $EXEFOUND = t && $AUTOUPDATE = off ]]; then
logMessage "Existing ADCIRC(+SWAN) executables were successfully found."
ADCIRCDIR=$EXEPATH/work # <-- setting the value of ADCIRCDIR
return 0
fi
#
logMessage "ADCIRC(+SWAN) executables were not found. They will be (re)built."
#
# Check the code out of svn and into the specified directory.
# TODO: Deal with svn username/password.
if [[ ! -d $EXEPATH ]]; then
mkdir -p $EXEPATH 2>> $SYSLOG
fi
cd $EXEPATH 2>> $SYSLOG
#
# Check the source code out of the repository. TODO: Enable other sources
# of source code, e.g., a tar.gz file on the local file system.
logMessage "Retrieving source code."
# TODO: Figure out how/when to 'svn update' source code already in place if
# AUTOUPDATE is on.
svn checkout $SOURCEURL . >> build.log 2>> $SYSLOG
mv build.log $EXEPATH/work 2>> $SYSLOG
#
# Now build the ADCIRC and ADCIRC+SWAN executables.
cd $EXEPATH/work 2>> $SYSLOG
#
# Write properties file to record what this script is attempting to do
# and make it easy to look in the executables directory to see how the
# code was compiled.
echo "DEBUG : $DEBUG" > $PROPERTIES
echo "SWAN : $SWAN" >> $PROPERTIES
echo "NETCDF : $NETCDF" >> $PROPERTIES
echo "NETCDF4 : $NETCDF4" >> $PROPERTIES
echo "NETCDF4_COMPRESSION : $NETCDF4_COMPRESSION" >> $PROPERTIES
echo "XDMF : $XDMF" >> $PROPERTIES
echo "SOURCEURL : $SOURCEURL" >> $PROPERTIES
echo "AUTOUPDATE : $AUTOUPDATE" >> $PROPERTIES
echo "EXEBASEPATH : $EXEBASEPATH" >> $PROPERTIES
echo "SCRIPTDIR : $SCRIPTDIR" >> $PROPERTIES
echo "SWANMACROSINC : $SWANMACROSINC" >> $PROPERTIES
echo "ADCOPTIONS : $ADCOPTIONS" >> $PROPERTIES
echo "SYSLOG : $SYSLOG" >> $PROPERTIES
#
# Set the correct SWAN compiler flags for this HPC platform.
cp ../swan/$SWANMACROSINC ../swan/macros.inc 2>> $SYSLOG
#
# Build the executables using the settings listed in the platforms.sh file.
logMessage "Building executables."
for executable in adcirc padcirc padcswan adcprep hstime aswip; do
logMessage "Building ${executable}."
MAKECMDLINE="make $executable $ADCOPTIONS DEBUG=$DEBUG SWAN=$SWAN NETCDF=$NETCDF NETCDF4=$NETCDF4 NETCDF4_COMPRESSION=$NETCDF4_COMPRESSION XDMF=$XDMF"
echo "MAKECMDLINE is $MAKECMDLINE" >> build.log 2>> $SYSLOG
$MAKECMDLINE >> build.log 2>&1
if [[ $? == 0 ]]; then
logMessage "Successfully built ${executable}."
else
warn "Failed to build $EXEPATH/work/${executable}."
return 1
fi
done
# All executables were built successfully; set the value of ADCIRCDIR.
ADCIRCDIR=$EXEPATH/work
}
#
#
# subroutine to run adcprep, using a pre-prepped archive of fort.13,
# fort.14 and fort.18 files
prep()
{ ADVISDIR=$1 # directory containing the now/forecast runs for this cycle
INPUTDIR=$2 # directory where grid and nodal attribute files are found
ENSTORM=$3 # ensemble member (nowcast, storm1, storm5, etc)
START=$4 # coldstart or hotstart
FROMDIR=$5 # directory containing files to hotstart this run from
HPCENV=$6 # machine to run on (jade, desktop, queenbee, etc)
NCPU=$7 # number of CPUs to request in parallel jobs
PREPPEDARCHIVE=$8 # preprocessed fort.13 and fort.14 package
GRIDFILE=$9 # fulldomain grid
ACCOUNT=${10} # account to charge time to
OUTPUTOPTIONS="${11}" # contains list of args for appending files
HOTSTARTCOMP=${12} # fulldomain or subdomain
WALLTIME=${13} # HH:MM:SS format
HOTSTARTFORMAT=${14} # "binary" or "netcdf"
MINMAX=${15} # "continuous" or "reset"
HOTSWAN=${16} # "yes" or "no" to reinitialize SWAN only
NAFILE=${17} # full domain nodal attributes file
TIMESTAMP=`date +%d%b%Y:%H:%M:%S`
#
# set the name of the archive of preprocessed input files
PREPPED=$PREPPEDARCHIVE
if [[ $START = coldstart ]]; then
PREPPED=$HINDCASTARCHIVE
fi
# determine if there is an archive of preprocessed input files
HAVEARCHIVE=yes
if [[ ! -e ${INPUTDIR}/${PREPPED} ]]; then
HAVEARCHIVE=no
fi
# create directory to run in
if [ ! -d $ADVISDIR/$ENSTORM ]; then
mkdir $ADVISDIR/$ENSTORM 2>> ${SYSLOG}
fi
echo "$TIMESTAMP adcprep.log entry for $FILE for ensemble member $ENSTORM in $ADVISDIR as follows: " >> $ADVISDIR/$ENSTORM/adcprep.log
cd $ADVISDIR/$ENSTORM
logMessage "Linking to full domain input files."
# symbolically link grid
if [ ! -e $ADVISDIR/$ENSTORM/fort.14 ]; then
ln -s $INPUTDIR/$GRIDFILE $ADVISDIR/$ENSTORM/fort.14 2>> ${SYSLOG}
fi
# symbolically link nodal attributes
if [ ! -e $ADVISDIR/$ENSTORM/fort.13 ]; then
if [[ ! -z $NAFILE && $NAFILE != null ]]; then
ln -s $INPUTDIR/$NAFILE $ADVISDIR/$ENSTORM/fort.13 2>> ${SYSLOG}
fi
fi
fi
if [[ $HAVEARCHIVE = yes ]]; then
# copy in the files that have already been preprocessed
logMessage "Copying input files that have already been decomposed."
cp $INPUTDIR/${PREPPED} . 2>> ${SYSLOG}
gunzip -f ${PREPPED} 2>> ${SYSLOG}
# untar the uncompressed archive
UNCOMPRESSEDARCHIVE=${PREPPED%.gz}
tar xvf $UNCOMPRESSEDARCHIVE > untarred_files.log 2>> ${SYSLOG}
logMessage "Removing $UNCOMPRESSEDARCHIVE"
rm $UNCOMPRESSEDARCHIVE 2>> ${SYSLOG}
fi
#
# this is a C O L D S T A R T
if [ $START = coldstart ]; then
# if we have variable river flux, link the fort.20 and fort.88 files
if [[ $VARFLUX = on || $VARFLUX = default ]]; then
# jgf20110525: For now, just copy a static file to this location
# and adcprep it. TODO: When real time flux data become available,
# grab those instead of relying on a static file.
ln -s ${INPUTDIR}/${HINDCASTRIVERFLUX} ./fort.20
# run adcprep to decompose the river elevation init (fort.88) file
ln -s ${INPUTDIR}/${RIVERINIT} ./fort.88
fi
# now run adcprep to decompose the files
if [[ $HAVEARCHIVE = no ]]; then
logMessage "Running adcprep to partition the mesh for $NCPU compute processors."
prepFile partmesh $NCPU $ACCOUNT $WALLTIME
logMessage "Running adcprep to prepare all files."
prepFile prepall $NCPU $ACCOUNT $WALLTIME
else
logMessage "Running adcprep to prepare new fort.15 file."
prepFile prep15 $NCPU $ACCOUNT $WALLTIME
if [[ $VARFLUX = on || $VARFLUX = default ]]; then
logMessage "Running adcprep to prepare new fort.20 file."
prepFile prep20 $NCPU $ACCOUNT $WALLTIME
logMessage "Running adcprep to prepare fort.88 file."
prepFile prep88 $NCPU $ACCOUNT $WALLTIME
fi
fi
else
# this is a H O T S T A R T
#
# copy in the swaninit file which contains the name of the swan
# control file (conventionally named fort.26 when used with ADCIRC)
if [[ $WAVES = on ]]; then
cp $INPUTDIR/swaninit.template $ADVISDIR/$ENSTORM/swaninit 2>> ${SYSLOG}
fi
# jgfdebug: TODO: FIXME: Hardcoded the time varying weirs input file
if [ -e $INPUTDIR/time-bonnet.in ]; then
logMessage "Copying $INPUTDIR/time-bonnet.in to $ADVISDIR/$ENSTORM."
cp $INPUTDIR/time-bonnet.in $ADVISDIR/$ENSTORM 2>> ${SYSLOG}
fi
# run adcprep to decompose the new files
if [[ $HAVEARCHIVE = no ]]; then
logMessage "Running adcprep to partition the mesh for $NCPU compute processors."
prepFile partmesh $NCPU $ACCOUNT $WALLTIME
logMessage "Running adcprep to prepare all files."
prepFile prepall $NCPU $ACCOUNT $WALLTIME
else
logMessage "Running adcprep to prepare new fort.15 file."
prepFile prep15 $NCPU $ACCOUNT $WALLTIME
if [[ $VARFLUX = on || $VARFLUX = default ]]; then
logMessage "Running adcprep to prepare new fort.20 file."
prepFile prep20 $NCPU $ACCOUNT $WALLTIME
fi
if [[ $WAVES = on ]]; then
PE=0
format="%04d"
while [[ $PE -lt $NCPU ]]; do
PESTRING=`printf "$format" $PE`
ln -s $ADVISDIR/$ENSTORM/fort.26 $ADVISDIR/$ENSTORM/PE${PESTRING}/fort.26 2>> ${SYSLOG}
PE=`expr $PE + 1`
done
fi
fi
logMessage "Copying existing output files to this directory."
if [[ $MINMAX = continuous ]]; then
# copy max and min files so that the max values will be
# preserved across hotstarts
for file in maxele.63 maxwvel.63 minpr.63 maxrs.63 maxvel.63 elemaxdry.63 nodeflag.63 rising.63 tinun.63; do
if [ -e $FROMDIR/$file ]; then
logMessage "Copying $FROMDIR/$file to $ADVISDIR/$ENSTORM/$file so that its values will be preserved across the hotstart."
cp $FROMDIR/$file $ADVISDIR/$ENSTORM/$file 2>> ${SYSLOG}
fi
done
else
logMessage "MINMAX was set to '$MINMAX' in the ASGS config file; as a result, the maxele.63 etc files will not be from the previous run to the current run. ADCIRC will start the record of max and min values anew."
fi
# copy existing fulldomain files if they are supposed to be appended
for file in fort.61 fort.62 fort.63 fort.64 fort.71 fort.72 fort.73 fort.74; do
matcharg=--${file/./}append
if [[ $file = "fort.71" || $file = "fort.72" ]]; then
matcharg="--fort7172append"
elif [[ $file = "fort.73" || $file = "fort.74" ]]; then
matcharg="--fort7374append"
fi
# check the output options to see if the file is being appended
for arg in $OUTPUTOPTIONS ; do
if [[ $matcharg = $arg ]]; then
# the file is being appended; check to see if it is in netcdf
# format, nd if so, use the netcdf name
netCDFArg=${matcharg/append/netcdf/}
for outputArg in $OUTPUTOPTIONS ; do
if [[ $outputArg = $netCDFArg ]]; then
file=${file/./}.nc
fi
done
if [ -e $FROMDIR/$file ]; then
logMessage "Copying $FROMDIR/$file to $ADVISDIR/$ENSTORM/$file so that it will be appended during the upcoming run."
cp $FROMDIR/$file $ADVISDIR/$ENSTORM/$file 2>> ${SYSLOG}
fi
fi
done
done
# bring in hotstart file(s)
if [[ $HOTSTARTCOMP = fulldomain ]]; then
if [[ $HOTSTARTFORMAT = netcdf ]]; then
# copy netcdf file so we overwrite the one that adcprep created
cp --remove-destination $FROMDIR/fort.67.nc $ADVISDIR/$ENSTORM/fort.68.nc >> $SYSLOG 2>&1
else
ln -s $FROMDIR/PE0000/fort.67 $ADVISDIR/$ENSTORM/fort.68 >> $SYSLOG 2>&1
fi
fi
if [[ $HOTSTARTCOMP = subdomain || $WAVES = on ]]; then
logMessage "Starting copy of subdomain hotstart files."
# copy the subdomain hotstart files over
# subdomain hotstart files are always binary formatted
PE=0
format="%04d"
while [ $PE -lt $NCPU ]; do
PESTRING=`printf "$format" $PE`
if [[ $HOTSTARTCOMP = subdomain ]]; then
cp $FROMDIR/PE${PESTRING}/fort.67 $ADVISDIR/$ENSTORM/PE${PESTRING}/fort.68 2>> ${SYSLOG}
fi
if [[ $WAVES = on && $HOTSWAN = on ]]; then
cp $FROMDIR/PE${PESTRING}/swan.67 $ADVISDIR/$ENSTORM/PE${PESTRING}/swan.68 2>> ${SYSLOG}
fi
PE=`expr $PE + 1`
done
logMessage "Completed copy of subdomain hotstart files."
fi
fi
# if we don't have an archive of our preprocessed files, create
# one so that we don't have to do another prepall
if [[ $HAVEARCHIVE = no ]]; then
logMessage "Creating an archive of preprocessed files and saving to ${INPUTDIR}/${PREPPED} to avoid having to run prepall again."
FILELIST='partmesh.txt PE*/fort.14 PE*/fort.18'
if [[ ! -z $NAFILE && $NAFILE != null ]]; then
FILELIST='partmesh.txt PE*/fort.14 PE*/fort.18 PE*/fort.13'
fi
tar cvzf ${INPUTDIR}/${PREPPED} ${FILELIST} 2>> ${SYSLOG}
# check status of tar operation; if it failed, delete the file
# it attempted to make and alert the operator
if [[ $? != 0 ]]; then
warn "The construction of a tar archive of the preprocessed input files has failed."
rm ${INPUTDIR}/${PREPPED} 2>> ${SYSLOG} 2>&1
fi
fi
}
#
# function to run adcprep in a platform dependent way to decompose
# the fort.15, fort.20, or fort.88 file
prepFile()
{ JOBTYPE=$1
NCPU=$2
ACCOUNT=$3
WALLTIME=$4
#
case $QUEUESYS in
"PBS")
QSCRIPTOPTIONS="--jobtype $JOBTYPE --ncpu $NCPU --ppn $PPN --queuename $SERQUEUE --account $ACCOUNT --walltime $WALLTIME --adcircdir $ADCIRCDIR --advisdir $ADVISDIR --qscript $SCRIPTDIR/input/machines/$HPCENV/$PREPCONTROLSCRIPT --enstorm ${ENSTORM} --notifyuser $NOTIFYUSER --syslog $SYSLOG"
perl $SCRIPTDIR/$QSCRIPTGEN $QSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.pbs 2>> ${SYSLOG}
# submit adcprep job, check to make sure qsub succeeded, and if not, retry
while [ true ]; do
qsub $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.pbs >> ${SYSLOG} 2>&1
if [[ $? = 0 ]]; then
break # qsub returned a "success" status
else
warn "qsub $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.pbs failed; will retry in 60 seconds."
sleep 60
fi
done
monitorJobs $QUEUESYS ${JOBTYPE}.${ENSTORM} $WALLTIME
logMessage "Finished adcprepping file ($JOBTYPE)."
;;
"SLURM")
QSCRIPTOPTIONS="--jobtype $JOBTYPE --ncpu $NCPU --ppn $PPN --queuename $SERQUEUE --account $ACCOUNT --walltime $WALLTIME --adcircdir $ADCIRCDIR --advisdir $ADVISDIR --qscript $SCRIPTDIR/input/machines/$ENV/$PREPCONTROLSCRIPT --enstorm ${ENSTORM} --notifyuser $NOTIFYUSER --syslog $SYSLOG"
#jgfdebug
logMessage "Preparing queue script for adcprep with the following: perl $SCRIPTDIR/$QSCRIPTGEN $QSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.slurm 2>> ${SYSLOG}"
perl $SCRIPTDIR/$QSCRIPTGEN $QSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.slurm 2>> ${SYSLOG}
# submit adcprep job, check to make sure sbatch succeeded, and if not, retry
while [ true ]; do
sbatch $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.slurm >> ${SYSLOG} 2>&1
if [[ $? = 0 ]]; then
break # qsub returned a "success" status
else
warn "sbatch $ADVISDIR/$ENSTORM/adcprep.${JOBTYPE}.slurm failed; will retry in 60 seconds."
sleep 60
fi
done
monitorJobs $QUEUESYS ${JOBTYPE}.${ENSTORM} $WALLTIME
logMessage "Finished adcprepping file ($JOBTYPE)."
;;
"SGE")
cd $ADVISDIR/$ENSTORM 2>> ${SYSLOG}
SERQSCRIPTOPTIONS="--jobtype $JOBTYPE --ncpu $NCPU --account $ACCOUNT --adcircdir $ADCIRCDIR --walltime $WALLTIME --advisdir $ADVISDIR --enstorm $ENSTORM --notifyuser $NOTIFYUSER --serqscript $SCRIPTDIR/input/machines/$HPCENV/$SERQSCRIPT"
perl $SCRIPTDIR/$SERQSCRIPTGEN $SERQSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/adcprep.serial.sge 2>> ${SYSLOG}
logMessage "Submitting $ADVISDIR/$ENSTORM/adcprep.serial.sge."
qsub $ADVISDIR/$ENSTORM/adcprep.serial.sge >> ${SYSLOG} 2>&1
# if qsub succeeded, monitor the job, otherwise an error is indicated
if [[ $? = 1 ]]; then
rangerResubmit $ADVISDIR $ENSTORM adcprep.serial.sge $SYSLOG
fi
# check once per minute until all jobs have finished
monitorJobs $QUEUESYS ${JOBTYPE}.${ENSTORM} $WALLTIME
allMessage "adcprep finished."
;;
*)
logMessage "Submitting job with $ADCIRCDIR/adcprep --np $NCPU --${JOBTYPE} >> $ADVISDIR/$ENSTORM/adcprep.log 2>&1"
$ADCIRCDIR/adcprep --np $NCPU --${JOBTYPE} >> $ADVISDIR/$ENSTORM/adcprep.log 2>&1
# check to see if adcprep completed successfully
if [[ $? != 0 ]]; then
error "The adcprep ${JOBTYPE} job failed. See the file $ADVISDIR/$ENSTORM/${JOBTYPE}.adcprep.log for details."
echo "The adcprep ${JOBTYPE} job failed. See the file $ADVISDIR/$ENSTORM/${JOBTYPE}.adcprep.log for details." >> jobFailed
fi
;;
esac
}
#
# subroutine that calls an external script over and over until it
# pulls down a new advisory (then it returns)
downloadCycloneData()
{ STORM=$1
YEAR=$2
RUNDIR=$3
SCRIPTDIR=$4
OLDADVISDIR=$5
TRIGGER=$6
ADVISORY=$7
FTPSITE=$8
RSSSITE=$9
FDIR=${10}
HDIR=${11}
STATEFILE=${12}
# activity_indicator "Checking remote site for new advisory..." &
echo "Checking remote site for new advisory..."
# pid=$!; trap "stop_activity_indicator ${pid}; exit" EXIT
cd $RUNDIR 2>> ${SYSLOG}
newAdvisory=false
newAdvisoryNum=null
forecastFileName=al${STORM}${YEAR}.fst
hindcastFileName=bal${STORM}${YEAR}.dat
# check to see if we have a leftover forecast.properties file from
# a previous advisory laying around here in our run directory, and if
# so, delete it
if [[ -e forecast.properties ]]; then
rm forecast.properties 2>> ${SYSLOG}
fi
OPTIONS="--storm $STORM --year $YEAR --ftpsite $FTPSITE --fdir $FDIR --hdir $HDIR --rsssite $RSSSITE --trigger $TRIGGER --adv $ADVISORY"
logMessage "Options for get_atcf.pl are as follows : $OPTIONS"
if [ "$START" = coldstart ]; then
logMessage "Downloading initial hindcast/forecast."
else
logMessage "Checking remote site for new advisory..."
fi
while [ $newAdvisory = false ]; do
if [ $TRIGGER != atcf ]; then
newAdvisoryNum=`perl $SCRIPTDIR/get_atcf.pl $OPTIONS 2>> $SYSLOG`
fi
# check to see if we have a new one, and if so, determine the
# new advisory number correctly
case $TRIGGER in
"atcf")
# if the forecast is already in ATCF format, then simply copy it
# to the run directory
cp $HDIR/$hindcastFileName . 2>> ${SYSLOG}
cp $FDIR/$forecastFileName . 2>> ${SYSLOG}
linkTarget=`readlink $FDIR/$forecastFileName`
# assume the advisory number is the first two characters in the
# symbolic link target of the forecast file name
newAdvisoryNum=${linkTarget:0:2}
if [ $newAdvisoryNum -gt $ADVISORY ]; then
newAdvisory="true"
else
newAdvisory="false"
fi
;;
"ftp")
if [ $START = hotstart ]; then
if ! diff $OLDADVISDIR/$forecastFileName ./$forecastFileName > /dev/null 2>> ${SYSLOG}; then
# forecasts from NHC ftp site do not have advisory number
newAdvisoryNum=`printf "%02d" $[$ADVISORY + 1]`
newAdvisory="true"
fi
fi
;;
"rss" | "rssembedded" )
# if there was a new advisory, the get_atcf.pl script
# would have returned the advisory number in stdout
if [[ ! -z $newAdvisoryNum && $newAdvisoryNum != null ]]; then
newAdvisory="true"
if [ -e $forecastFileName ]; then
mv $forecastFileName $forecastFileName.ftp 2>> $SYSLOG
fi
fi
;;
*)
fatal "Invalid 'TRIGGER' type: '$TRIGGER'; must be ftp, rss, rssembedded, or atcf."
;;
esac
if [ $START = coldstart ]; then
if [ $TRIGGER = ftp ]; then
newAdvisoryNum=$ADVISORY
fi
newAdvisory="true"
fi
if [[ $newAdvisory = false ]]; then
sleep 60 # we are hotstarting, the advisory is same as last one
fi
done
logMessage "New forecast detected."
cp -f $STATEFILE ${STATEFILE}.old
sed 's/ADVISORY=.*/ADVISORY='$newAdvisoryNum'/' $STATEFILE > ${STATEFILE}.new
cp -f ${STATEFILE}.new $STATEFILE >> ${SYSLOG} 2>&1
if [[ $TRIGGER = rss || $TRIGGER = rssembedded ]]; then
perl ${SCRIPTDIR}/nhc_advisory_bot.pl --input ${forecastFileName}.html --output $forecastFileName --metadata forecast.properties >> ${SYSLOG} 2>&1
fi
if [[ $FTPSITE = filesystem ]]; then
cp $HDIR/$hindcastFileName $hindcastFileName 2>> ${SYSLOG}
fi
}
#
# subroutine that polls an external ftp site for background meteorology data,
# converts it to OWI format (reprojecting the data if necessary), makes
# symbolic links to it, and returns.
downloadBackgroundMet()
{
RUNDIR=$1
SCRIPTDIR=$2
BACKSITE=$3
BACKDIR=$4
ENSTORM=$5
CSDATE=$6
HSTIME=$7
FORECASTLENGTH=$8
ALTNAMDIR=$9
FORECASTCYCLE=${10}
ARCHIVEBASE=${11}
ARCHIVEDIR=${12}
STATEFILE=${13}
#
cd $RUNDIR 2>> ${SYSLOG}
if [[ $ENSTORM != "nowcast" ]]; then
grep ADVISORY $STATEFILE | sed 's/ADVISORY.*=//' | sed 's/^\s//' > currentCycle 2>> ${SYSLOG}
fi
newAdvisoryNum=0
while [[ $newAdvisoryNum -lt 2 ]]; do
OPTIONS="--rundir $RUNDIR --backsite $BACKSITE --backdir $BACKDIR --enstorm $ENSTORM --csdate $CSDATE --hstime $HSTIME --forecastlength $FORECASTLENGTH --altnamdir $ALTNAMDIR --scriptdir $SCRIPTDIR --forecastcycle $FORECASTCYCLE --archivedruns ${ARCHIVEBASE}/${ARCHIVEDIR}"
newAdvisoryNum=`perl ${SCRIPTDIR}/get_nam.pl $OPTIONS 2>> ${SYSLOG}`
if [[ $newAdvisoryNum -lt 2 ]]; then
sleep 60
fi
done
# record the new advisory number to the statefile
cp -f $STATEFILE ${STATEFILE}.old 2>> ${SYSLOG}
sed 's/ADVISORY=.*/ADVISORY='$newAdvisoryNum'/' $STATEFILE > ${STATEFILE}.new
cp -f ${STATEFILE}.new $STATEFILE >> ${SYSLOG} 2>&1
}
#
# subroutine that downloads river flux data from an external ftp site
# and constructs a river flux boundary condition file (fort.20) to covert
# the full time period of the run
downloadRiverFluxData()
{
ADVISDIR=$1
MESHFILE=$2
RIVERSITE=$3
RIVERDIR=$4
RIVERUSER=$5
RIVERDATAPROTOCOL=$6
ENSTORM=$7
CSDATE=$8
HSTIME=$9
SCRIPTDIR=${10}
DEFAULTFILE=${11}
USERIVERFILEONLY=${12}
#
OPTIONS="--advisdir $ADVISDIR --meshfile $MESHFILE --riversite $RIVERSITE --riverdir $RIVERDIR --riveruser $RIVERUSER --riverdataprotocol $RIVERDATAPROTOCOL --enstorm $ENSTORM --csdate $CSDATE --hstime $HSTIME --scriptdir $SCRIPTDIR --defaultfile $DEFAULTFILE"
TRIES=0
SUCCESS=no
if [[ $USERIVERFILEONLY = no ]]; then
while [[ $TRIES -lt 2 ]]; do
perl ${SCRIPTDIR}/get_flux.pl $OPTIONS 2>> ${SYSLOG}
if [[ $? = 0 ]]; then
logMessage "Completed construction of river flux boundary condition (fort.20 file)."
SUCCESS=yes
break
else
TRIES=$[$TRIES + 1]
warn "Attempt $TRIES at constructing river flux boundary condition (fort.20) file has failed. After 2 attempts, the default flux boundary condition file '$DEFAULTFILE' will be used."
sleep 60
fi
done
fi
if [[ $SUCCESS = no ]]; then
error "Using default river flux boundary condition file '$DEFAULTFILE'."
ln -s $DEFAULTFILE ./fort.20 2>> ${SYSLOG}
fi
}
#
# see if a task has been running longer than a specified time limit
checkTimeLimit()
{
STARTTIME=$1
TIMELIMIT=$2
#
# convert time limit to seconds, assuming it is in the format HH:MM:SS
hours=${TIMELIMIT:0:2} # requires leading zero! e.g., 05:00:00
minutes=${TIMELIMIT:3:2}
seconds=${TIMELIMIT:6:2}
# bash interprets numbers with leading zeroes as octal ... the 10# prefix
# tells bash that the numbers are base 10
limit=$((10#$hours * 3600 + 10#$minutes * 60 + 10#$seconds)) # in seconds
endTime=`date +%s`
runTime=$(($endTime - $STARTTIME))
if [[ $runTime -gt $limit ]]; then
hoursEnd=$(($limit / 3600))
remainder=$(($limit % 3600))
minutesEnd=$(($remainder / 60))
secondsEnd=$(($remainder % 60))
format="%02d:%02d:%02d"
hms=`printf "$format" $hoursEnd $minutesEnd $secondsEnd`
warn "The time limit is $TIMELIMIT but the total time used so far is $hms. Therefore, the time limit has been exceeded."
return 1
else
return 0
fi
}
#
# watches for the existence of certain files that are written by the job as
# it executes and proceeds according to the status that is indicated by
# those files
monitorJobs()
{ QUEUESYS=$1
ENSTORM_TEMP=$2
WALLTIME=$3
#
logMessage "Waiting for $ENSTORM_TEMP job to start."
until [[ -e ${ENSTORM_TEMP}.run.start ]]; do
sleep 10
done
logMessage "The $ENSTORM_TEMP job has started."
startTime=`date +%s` # epoch seconds
until [[ -e ${ENSTORM_TEMP}.run.finish || -e ${ENSTORM_TEMP}.run.error ]]; do
sleep 10
if ! checkTimeLimit $startTime $WALLTIME ; then
echo "The ${ENSTORM_TEMP} job exceeded its wall clock time limit of '$WALLTIME'." > ${ENSTORM_TEMP}.run.error
# if this job was submitted by mpiexec, then terminate it; otherwise,
# it could run for a long time, delaying the continued execution
# of the ASGS (the ASGS won't start the next cycle until all forecast
# ensemble members in the current cycle have completed); this also
# prevents cpus from being tied up unnecessarily ...
# if the job was submitted through a queueing system, then the
# queueing system will terminate it
if [[ $QUEUESYS = mpiexec ]]; then
pid=`grep 'mpiexec subshell pid' ${ADVISDIR}/${ENSTORM}/run.properties | sed 's/mpiexec subshell pid.*://' | sed 's/^\s//'`
#logMessage "Terminating the $ENSTORM_TEMP job with the command 'kill -TERM `ps --ppid $pid -o pid --no-headers'."
# need to kill the mpiexec process, but don't know its process ID
# ... but we do have the process ID of its parent subshell
kill -TERM `ps --ppid $pid -o pid --no-headers` >> ${SYSLOG} 2>&1
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S'`
logMessage "$ENSTORM_TEMP job in $PWD terminated by ASGS for exceeding expected wall clock time." >> ${ENSTORM_TEMP}.run.error
else
# if we are over the wall clock limit, wait until the operating system has had a chance
# to write the job log file, or until 5 minutes have passed
overLimitTime=`date +%s`
until [[ ! -e ${ENSTORM_TEMP}.out ]]; do
logMessage "Waiting for queueing system to write out the job log file ${ENSTORM_TEMP}.out."
sleep 60
nowTime=`date +%s`
if [[ `expr $nowTime - $overLimitTime` -gt 300 ]]; then
warn "After 5 minutes, the ${ENSTORM_TEMP}.out file did not appear. Proceeding with error recovery."
break
fi
done
fi
fi
done
if [[ -e ${ENSTORM_TEMP}.run.error ]]; then
error "The $ENSTORM_TEMP run failed; results are not available for this ensemble member for this advisory."
cat ${ENSTORM_TEMP}.run.error >> jobFailed
fi
if [[ -e ${ENSTORM_TEMP}.run.finish ]]; then
logMessage "The $ENSTORM_TEMP job appears to have run to completion successfully."
fi
logMessage "Finished monitoring $ENSTORM_TEMP job."
}
#
# submits a job to the local queueing system
submitJob()
{ QUEUESYS=$1
NCPU=$2
ADCIRCDIR=$3
ADVISDIR=$4
SCRIPTDIR=$5
INPUTDIR=$6
ENSTORM=$7
NOTIFYSER=$8
HPCENV=$9
ACCOUNT=${10}
PPN=${11}
NUMWRITERS=${12}
HOTSTARTCOMP=${13}
WALLTIME=${14}
JOBTYPE=${15}
#
#
CLOPTIONS="" # command line options
LOCALHOTSTART=""
CPUREQUEST=$NCPU
if [[ $NUMWRITERS != "0" ]]; then
CLOPTIONS="-W $NUMWRITERS"
CPUREQUEST=`expr $NCPU + $NUMWRITERS`
fi
# record the number of requested CPUs for use in determining capacity to run another job
echo "cpurequest : $CPUREQUEST" >> ${ADVISDIR}/${ENSTORM}/run.properties
echo "ncpu : $NCPU" >> ${ADVISDIR}/${ENSTORM}/run.properties # number of compute CPUs
echo "numwriters : $NUMWRITERS" >> ${ADVISDIR}/${ENSTORM}/run.properties # number of dedicated writer CPUs
if [[ $HOTSTARTCOMP = subdomain ]]; then
CLOPTIONS="${CLOPTIONS} -S"
LOCALHOTSTART="--localhotstart"
fi
case $QUEUESYS in
#
# Load Sharing Facility (LSF); used on topsail at UNC
"LSF")
bsub -x -n $NCPU -q $QUEUENAME -o log.%J -e err.%J -a mvapich mpirun $ADCIRCDIR/padcirc $CLOPTIONS >> ${SYSLOG}
;;
#
# LoadLeveler (often used on IBM systems)
"LoadLeveler")
perl $SCRIPTDIR/loadleveler.pl --jobtype $JOBTYPE --ncpu $NCPU --adcircdir $ADCIRCDIR --advisdir $ADVISDIR --inputdir $INPUTDIR --enstorm $ENSTORM --notifyuser $NOTIFYUSER --numwriters $NUMWRITERS $LOCALHOTSTART > $ADVISDIR/$ENSTORM/${JOBTYPE}.ll 2>> ${SYSLOG}
llsubmit $ADVISDIR/$ENSTORM/${JOBTYPE}.ll >> ${SYSLOG} 2>&1
;;
#
# Portable Batch System (PBS); widely used
"PBS")
QSCRIPTOPTIONS="--jobtype $JOBTYPE --ncpu $NCPU --queuename $QUEUENAME --account $ACCOUNT --adcircdir $ADCIRCDIR --advisdir $ADVISDIR --qscript $SCRIPTDIR/input/machines/$HPCENV/$QSCRIPT --enstorm $ENSTORM --notifyuser $NOTIFYUSER --walltime $WALLTIME --submitstring $SUBMITSTRING $LOCALHOTSTART --syslog $SYSLOG"
if [[ $PPN -ne 0 ]]; then
QSCRIPTOPTIONS="$QSCRIPTOPTIONS --ppn $PPN"
fi
if [[ $NUMWRITERS != "0" ]]; then
QSCRIPTOPTIONS="$QSCRIPTOPTIONS --numwriters $NUMWRITERS"
fi
logMessage "QSCRIPTOPTIONS is $QSCRIPTOPTIONS"
perl $SCRIPTDIR/$QSCRIPTGEN $QSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/${JOBTYPE}.pbs 2>> ${SYSLOG}
logMessage "Submitting $ADVISDIR/$ENSTORM/${JOBTYPE}.pbs"
# submit job, check to make sure qsub succeeded, and if not, retry
while [ true ]; do
qsub $ADVISDIR/$ENSTORM/${JOBTYPE}.pbs >> ${SYSLOG} 2>&1
if [[ $? = 0 ]]; then
break # qsub returned a "success" status
else
warn "qsub $ADVISDIR/$ENSTORM/${JOBTYPE}.pbs failed; will retry in 60 seconds."
sleep 60
fi
done
;;
#
# SLURM
"SLURM")
QSCRIPTOPTIONS="--jobtype $JOBTYPE --ncpu $NCPU --queuename $QUEUENAME --account $ACCOUNT --adcircdir $ADCIRCDIR --advisdir $ADVISDIR --qscript $SCRIPTDIR/input/machines/$HPCENV/$QSCRIPT --enstorm $ENSTORM --notifyuser $NOTIFYUSER --walltime $WALLTIME --submitstring $SUBMITSTRING $LOCALHOTSTART --syslog $SYSLOG"
if [[ $PPN -ne 0 ]]; then
QSCRIPTOPTIONS="$QSCRIPTOPTIONS --ppn $PPN"
fi
if [[ $NUMWRITERS != "0" ]]; then
QSCRIPTOPTIONS="$QSCRIPTOPTIONS --numwriters $NUMWRITERS"
fi
logMessage "QSCRIPTOPTIONS is $QSCRIPTOPTIONS"
perl $SCRIPTDIR/$QSCRIPTGEN $QSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/${JOBTYPE}.slurm 2>> ${SYSLOG}
logMessage "Submitting $ADVISDIR/$ENSTORM/${JOBTYPE}.slurm"
# submit job, check to make sure qsub succeeded, and if not, retry
while [ true ]; do
sbatch $ADVISDIR/$ENSTORM/${JOBTYPE}.slurm >> ${SYSLOG} 2>&1
if [[ $? = 0 ]]; then
break # sbatch returned a "success" status
else
warn "sbatch $ADVISDIR/$ENSTORM/${JOBTYPE}.slurm failed; will retry in 60 seconds."
sleep 60
fi
done
;;
#
# No queueing system, just mpiexec (used on standalone computers)
"mpiexec")
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S'`
echo "[${DATETIME}] Starting ${JOBTYPE}.${ENSTORM} job in $PWD." >> ${ADVISDIR}/${ENSTORM}/${JOBTYPE}.${ENSTORM}.run.start
logMessage "Submitting job via $SUBMITSTRING $CPUREQUEST $ADCIRCDIR/$JOBTYPE $CLOPTIONS >> ${SYSLOG} 2>&1"
# submit the parallel job in a subshell
(
$SUBMITSTRING $CPUREQUEST $ADCIRCDIR/$JOBTYPE $CLOPTIONS >> ${ADVISDIR}/${ENSTORM}/adcirc.log 2>&1
ERROVALUE=$?
RUNSUFFIX="finish"
DATETIME=`date +'%Y-%h-%d-T%H:%M:%S'`
if [ $ERROVALUE != 0 ] ; then
RUNSUFFIX="error"
fi
echo "[${DATETIME}] Finished ${JOBTYPE}.${ENSTORM} job in $PWD with return value = $ERROVALUE." >> ${ADVISDIR}/${ENSTORM}/${JOBTYPE}.${ENSTORM}.run.${RUNSUFFIX}
) &
# write the process id for mpiexec to the run.properties file so that monitorJobs()
# can kill the job if it exceeds the expected wall clock time
echo "mpiexec subshell pid : $!" >> ${ADVISDIR}/${ENSTORM}/run.properties 2>> ${SYSLOG}
;;
#
# Sun Grid Engine (SGE); used on Sun and many Linux clusters
"SGE")
QSCRIPTOPTIONS="--jobtype $JOBTYPE --ncpu $NCPU --ncpudivisor $NCPUDIVISOR --queuename $QUEUENAME --account $ACCOUNT --adcircdir $ADCIRCDIR --advisdir $ADVISDIR --qscript $SCRIPTDIR/input/machines/$HPCENV/$QSCRIPT --enstorm $ENSTORM --notifyuser $NOTIFYUSER --walltime $WALLTIME --submitstring $SUBMITSTRING --syslog $SYSLOG --numwriters $NUMWRITERS $LOCALHOTSTART"
perl $SCRIPTDIR/$QSCRIPTGEN $QSCRIPTOPTIONS > $ADVISDIR/$ENSTORM/padcirc.sge 2>> ${SYSLOG}
logMessage "Submitting $ADVISDIR/$ENSTORM/padcirc.sge"
qsub $ADVISDIR/$ENSTORM/padcirc.sge >> ${SYSLOG} 2>&1
# if qsub failed, resubmit the job 5 times before giving up
if [[ $? = 1 ]]; then
rangerResubmit $ADVISDIR $ENSTORM ${JOBTYPE}.sge $SYSLOG
fi
;;
*)
fatal "Queueing system $QUEUESYS unrecognized."
;;
esac
}
#
# since valid jobs are sometimes rejected on ranger, this function will
# attempt to resubmit rejected jobs 5 times before giving up
rangerResubmit()
{
ADVISDIR=$1
ENSTORM=$2
SCRIPTNAME=$3
SYSLOG=$4
#
num_retries=0
success=0
while [[ $num_retries -le 5 ]]; do