Imaging scripts for 15A w/contsub

Astroua · Nov 30, 2019 · 790aeee · 790aeee
1 parent 860bc91
commit 790aeee
Show file tree

Hide file tree

Showing 3 changed files with 286 additions and 0 deletions.
diff --git a/15A-175/HI/imaging/HI_imaging_array_submission_04kms_nocontsub.sh b/15A-175/HI/imaging/HI_imaging_array_submission_04kms_nocontsub.sh
@@ -0,0 +1,110 @@
+#!/bin/bash
+#SBATCH --time=40:00:00
+#SBATCH --mem=128000M
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=32
+#SBATCH --job-name=M31_15A_042kms_nocontsub-%A-%a
+#SBATCH --output=casa-M31_15A_042kms_nocontsub-%A-%a.out
+#SBATCH --array=0-190
+
+# Stage 2
+# --array=0-95
+
+# Use array to set which channels will get imaged.
+# Run from a separate folder so the log files are in one place.
+
+module restore my_default
+
+source /home/ekoch/.bashrc
+source /home/ekoch/preload.bash
+
+job_num=$SLURM_ARRAY_TASK_ID
+
+# Build in a slight offset for each job to avoid starting a bunch of CASA
+# sessions at once
+# sleep $(($job_num + 30))
+
+# Move to scratch space b/c casa write out the temporary files into the same folder
+export scratch_path=/home/ekoch/scratch/M31_imaging/15A-175/
+
+cd $scratch_path
+
+Xvfb :1 &
+export DISPLAY=:1
+
+# Set which imaging stage to run: the initial 5-sigma clean (1) or the full clean with a mask to (2)
+stage=1
+
+if (( $stage==1 )); then
+    script_name="${HOME}/code/LocalGroup-VLA/15A-175/HI/imaging/HI_single_channel_clean.py"
+    # Parameter file for tclean
+    param_file="/home/ekoch/code/LocalGroup-VLA/15A-175/HI/imaging/param_files/15A_B_C_14A_0_4kms_nocontsub.saved"
+
+    # Start 4 channels running on the node
+    # This is well-suited for the cedar base nodes
+    start_chan=$(($job_num * 4))
+    end_chan=$((($job_num + 1) * 4))
+
+elif (( $stage==2 )); then
+    script_name="${HOME}/code/LocalGroup-VLA/15A-175/HI/imaging/HI_single_channel_clean_stage2.py"
+    param_file="/home/ekoch/code/LocalGroup-VLA/15A-175/HI/imaging/param_files/15A_B_C_14A_0_4kms_nocontsub_stage2.saved"
+
+    # Start 8 channels running on the node
+    # cleaning takes most of the time but uses fewer CPUs
+    # overload cpu usage
+    # This is well-suited for the cedar base nodes
+    start_chan=$(($job_num * 8))
+    end_chan=$((($job_num + 1) * 8))
+
+else
+    echo "Stage must be 1 or 2, not ${stage}".
+    exit 1
+fi
+
+
+# Path to the casa files
+export casa_scratch_path="$HOME/scratch/casa-release-5.4.1-32.el7"
+
+export chan_path_name="HI_nocontsub_0_42kms"
+
+# B/c CASA spawns other processes internally, and if something crashes in the scripts,
+# the wait command may hang until the job is killed
+# Try recording the casa interpreter PIDs and only make wait subject to those.
+pids=
+
+for (( chan_num = $start_chan; chan_num < $end_chan; chan_num++ )); do
+
+    echo "Running channel "$chan_num
+
+    # Make a new directory on the node storage
+    tmp_dir=$SLURM_TMPDIR/run_chan_${chan_num}
+    mkdir $tmp_dir
+
+    cd $tmp_dir
+
+    # Move the data to the temp path
+    mkdir ${chan_path_name}
+    mkdir ${chan_path_name}/channel_${chan_num}
+    cp -r $scratch_path/${chan_path_name}/channel_${chan_num}/* ${chan_path_name}/channel_${chan_num}/
+
+    # Copy a new casa instance to avoid slower i/o on scratch or in home
+    cp -r $casa_scratch_path .
+
+    # Copy the init file
+    mkdir .casa
+    cp $HOME/.casa/init.py .casa/
+
+    rc_path="${tmp_dir}/.casa"
+
+    (casa-release-5.4.1-32.el7/bin/mpicasa -n 8 casa-release-5.4.1-32.el7/bin/casa --rcdir ${rc_path} --nologger --nogui --logfile $scratch_path/${chan_path_name}/casa_M31_HI_15A_04kms_${chan_num}_${SLURM_JOB_ID}_$(date "+%Y%m%d-%H%M%S")_stage${stage}.log --nocrashreport -c $script_name $chan_num $param_file "${chan_path_name}"; cp -r ${chan_path_name}/channel_${chan_num}/* $scratch_path/${chan_path_name}/channel_${chan_num}/) &
+    pids+=" $!"
+
+    cd $tmp_dir
+
+    sleep 5
+
+done
+
+wait $pids || { echo "There was an error" >&2; exit 1; }
+
+echo "All CASA jobs exited."
diff --git a/15A-175/HI/imaging/param_files/15A_B_C_14A_0_4kms_nocontsub.saved b/15A-175/HI/imaging/param_files/15A_B_C_14A_0_4kms_nocontsub.saved
@@ -0,0 +1,88 @@
+taskname           = "tclean"
+vis                =  '14A_15A_HI_LSRK.ms.contsub'
+selectdata         =  True
+field              =  "M31*"
+spw                =  "0,1"
+timerange          =  ""
+uvrange            =  ""
+antenna            =  ""
+scan               =  ""
+observation        =  ""
+intent             =  ""
+datacolumn         =  "corrected"
+imagename          =  "M31_15A_B_C_14A_HI_contsub_width_1_2kms"
+imsize             =  5250
+cell               =  "1.0arcsec"
+phasecenter        =  'J2000 00:44:33.854 +41d57m42.572'
+stokes             =  "I"
+projection         =  "SIN"
+startmodel         =  ""
+specmode           =  "mfs"
+reffreq            =  ""
+nchan              =  1
+start              =  1
+width              =  1
+outframe           =  "LSRK"
+veltype            =  "radio"
+restfreq           =  ""
+interpolation      =  "linear"
+perchanweightdensity = False
+gridder            =  "mosaic"
+facets             =  1
+psfphasecenter     =  ""
+chanchunks         =  1
+wprojplanes        =  1
+vptable            =  ""
+usepointing        =  False
+mosweight          =  True
+aterm              =  True
+psterm             =  False
+wbawp              =  True
+conjbeams          =  False
+cfcache            =  ""
+computepastep      =  360.0
+rotatepastep       =  360.0
+pblimit            =  0.05
+normtype           =  "flatnoise"
+deconvolver        =  "multiscale"
+scales             =  [0, 6, 12, 24, 60, 120, 240, 480]
+nterms             =  2
+smallscalebias     =  0.6
+restoration        =  True
+restoringbeam      =  []
+pbcor              =  False
+outlierfile        =  ""
+weighting          =  "natural"
+robust             =  0.5
+nsigma             =  0.0
+npixels            =  0
+uvtaper            =  []
+niter              =  1000000
+gain               =  0.1
+threshold          =  "2.9mJy/beam"
+cycleniter         =  500
+cyclefactor        =  1.0
+minpsffraction     =  0.05
+maxpsffraction     =  0.8
+interactive        =  False
+usemask            =  "auto-multithresh"
+mask               =  ""
+pbmask             =  0.0
+maskthreshold      =  ""
+maskresolution     =  ""
+nmask              =  0
+sidelobethreshold    =  2.5
+noisethreshold     =  3.5
+lownoisethreshold    =  2.0
+negativethreshold    =  0.0
+smoothfactor       =  3.0
+minbeamfrac        =  0.8
+cutthreshold       =  0.01
+growiterations     =  75
+fastnoise          =  False
+restart            =  True
+savemodel          =  "none"
+calcres            =  True
+calcpsf            =  True
+parallel           =  True
+
diff --git a/15A-175/HI/imaging/param_files/15A_B_C_14A_0_4kms_nocontsub_stage2.saved b/15A-175/HI/imaging/param_files/15A_B_C_14A_0_4kms_nocontsub_stage2.saved
@@ -0,0 +1,88 @@
+taskname           = "tclean"
+vis                =  '14A_15A_HI_LSRK.ms.contsub'
+selectdata         =  True
+field              =  "M31*"
+spw                =  "0,1"
+timerange          =  ""
+uvrange            =  ""
+antenna            =  ""
+scan               =  ""
+observation        =  ""
+intent             =  ""
+datacolumn         =  "corrected"
+imagename          =  "M31_15A_B_C_14A_HI_contsub_width_1_2kms"
+imsize             =  5250
+cell               =  "1.0arcsec"
+phasecenter        =  'J2000 00:44:33.854 +41d57m42.572'
+stokes             =  "I"
+projection         =  "SIN"
+startmodel         =  ""
+specmode           =  "mfs"
+reffreq            =  ""
+nchan              =  1
+start              =  1
+width              =  1
+outframe           =  "LSRK"
+veltype            =  "radio"
+restfreq           =  ""
+interpolation      =  "linear"
+perchanweightdensity = False
+gridder            =  "mosaic"
+facets             =  1
+psfphasecenter     =  ""
+chanchunks         =  1
+wprojplanes        =  1
+vptable            =  ""
+usepointing        =  False
+mosweight          =  True
+aterm              =  True
+psterm             =  False
+wbawp              =  True
+conjbeams          =  False
+cfcache            =  ""
+computepastep      =  360.0
+rotatepastep       =  360.0
+pblimit            =  0.05
+normtype           =  "flatnoise"
+deconvolver        =  "multiscale"
+scales             =  [0, 6, 12, 24, 60, 120, 240, 480]
+nterms             =  2
+smallscalebias     =  0.6
+restoration        =  True
+restoringbeam      =  []
+pbcor              =  False
+outlierfile        =  ""
+weighting          =  "natural"
+robust             =  0.5
+nsigma             =  0.0
+npixels            =  0
+uvtaper            =  []
+niter              =  1000000
+gain               =  0.1
+threshold          =  "2.9mJy/beam"
+cycleniter         =  500
+cyclefactor        =  1.0
+minpsffraction     =  0.05
+maxpsffraction     =  0.8
+interactive        =  False
+usemask            =  "pb"
+mask               =  None
+pbmask             =  0.05
+maskthreshold      =  ""
+maskresolution     =  ""
+nmask              =  0
+sidelobethreshold    =  2.5
+noisethreshold     =  3.5
+lownoisethreshold    =  2.0
+negativethreshold    =  0.0
+smoothfactor       =  3.0
+minbeamfrac        =  0.8
+cutthreshold       =  0.01
+growiterations     =  75
+fastnoise          =  False
+restart            =  True
+savemodel          =  "none"
+calcres            =  True
+calcpsf            =  True
+parallel           =  True
+