From 381422b00d946bf3fcfbf6e32bc84e98e72c1ede Mon Sep 17 00:00:00 2001 From: Yogesh Maan Date: Tue, 9 Feb 2021 01:48:09 +0100 Subject: [PATCH 1/4] Add opts for better control over RFI excision --- bin/crp_rficlean_fil.sh | 14 +++---- include/rficlean.h | 2 + src/cleanit.c | 89 ++++++++++++++++++++++++----------------- src/rficlean.c | 89 +++++++++++++++++++++++++++++------------ src/rficlean_data.c | 28 ++++++++++--- 5 files changed, 148 insertions(+), 74 deletions(-) diff --git a/bin/crp_rficlean_fil.sh b/bin/crp_rficlean_fil.sh index b4c7de2..5a197e8 100755 --- a/bin/crp_rficlean_fil.sh +++ b/bin/crp_rficlean_fil.sh @@ -1,6 +1,6 @@ #!/bin/bash #======================================================================= -# Crude parallelization of rfiClean for processing filterbank files. +# Crude parallelization of RFIClean for processing filterbank files. # # Yogesh Maan, Dec. 2019 #======================================================================= @@ -39,13 +39,13 @@ Nb=`echo "scale=0; $nsamples/($Np*$block) + 1" | bc` -echo "Starting parallel-rfiClean processing at: " +echo "Starting parallel-RFIClean processing at: " date echo "" ##-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= # prepare and execute the commands for all parts -mkdir -p rfiClean_ps +mkdir -p RFIClean_ps combine_cmd="cat " remove_cmd="rm " count=0 @@ -57,15 +57,15 @@ for (( c=1; c<=$Np; c++ )); do psfile=${rtag}_part${count}.ps #------------------------------------- if [ $count -eq 1 ]; then - cmd="rficlean -t $block $flag -o $outfile -ps rfiClean_ps/$psfile $infile -bst $bst -nbl $Nb &" + cmd="rficlean -t $block $flag -o $outfile -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb &" elif [ $count -eq $Np ]; then combine_cmd="${combine_cmd} ${tempout}" remove_cmd="${remove_cmd} ${tempout}" - cmd="rficlean -t $block $flag -o $tempout -ps rfiClean_ps/$psfile $infile -bst $bst -headerless &" + cmd="rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -headerless &" else combine_cmd="${combine_cmd} ${tempout}" remove_cmd="${remove_cmd} ${tempout}" - cmd="rficlean -t $block $flag -o $tempout -ps rfiClean_ps/$psfile $infile -bst $bst -nbl $Nb -headerless &" + cmd="rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb -headerless &" fi echo $cmd eval $cmd @@ -83,7 +83,7 @@ echo $cmd eval $cmd -echo "Finished parallel-rfiClean processing at: " +echo "Finished parallel-RFIClean processing at: " date echo "" diff --git a/include/rficlean.h b/include/rficlean.h index 5075908..2aec681 100644 --- a/include/rficlean.h +++ b/include/rficlean.h @@ -1,5 +1,6 @@ #include #include +#include #include #include "header.h" #include @@ -14,6 +15,7 @@ float *fftstat, *chanstat, *predist, *xpredist, *postdist, *xpostdist, *finaldis double *tfvar, *tfmean,meanvar,rmsvar ; double *chandata, *mspec, *rspec, *vspec, *wspec, *wt; long int *coff; +bool RFIx,rfiFDx,rfiTx,rfiSx,rfiMSx,rfiVSx,rfiSclip; fftw_complex *in; fftw_complex *out; diff --git a/src/cleanit.c b/src/cleanit.c index c898aa7..2425acb 100644 --- a/src/cleanit.c +++ b/src/cleanit.c @@ -371,11 +371,19 @@ void cleanit(float *data, int nchans, long int nadd) isame = all_samef(chandata,nadd); } if( isame == 0){ - fftclean(in,out,nadd,inc); // clean some periodic RFIs + if(rfiFDx){ + fftclean(in,out,nadd,inc); // clean some periodic RFIs + } for (ii = 0; ii0){ kk = (int)(0.9*nchans); for (i=kk;i - filterbank data file (def=stdin)"); - puts("-t - number of time samples in a block (for periodicity search; def=4096)"); + puts("-t - number of time samples in a block (def=4096)"); puts("-ft - specify threshold for FT-cleaning (def=6.0)"); - puts("-st - specify threshold for timeseries-cleaning (def=10.0)"); - puts("-rt - specify threshold for chan-diff cleaning (def=4.0)"); - puts("-n - specify output number of bits (def=input)"); puts("-white - whitten each spectrum before RFI excision"); - puts("-zeordm - remove 0-dm powers before writing out data"); - puts("-pcl - assume that input data may have some parts replaced"); - puts(" by 0s or mean/median by some other RFI excision program"); - puts("-nharm - Number of fundamental+harmonics to be removed"); + puts("-nharm - Number of fundamental+harmonics to be excised"); puts(" (def=1, i.e., only fundamental)"); - puts("-T - number of time samples to avg before output (def=0)"); - puts("-gm - For gmrt raw-data, name of the file that contains header information"); - puts("-gmtstamp - For gmrt data, name of the file that contains time-stamp information"); + puts("-st - specify threshold for timeseries cleaning (def=10.0)"); + puts("-rt - specify threshold for channel cleaning (def=4.0)"); + puts("-clt - specify threshold for channel clipping (def=5.0)"); puts("-ps - File-name of the output diagnostic plot (def=rficlean_output.ps)"); puts("-o - specify output filename (def=stdout)"); + puts(""); + puts("To safeguard a known periodic signal:"); puts("-psrf - fundamental freq. (Hz) of pulsar to be protected (def=none)"); - puts("-psrfdf - Delta-frequency (on either side) of the fundamental & harmonic"); - puts("-psrfbins - *Nbins (on either side) of the fundamental & harmonic"); - puts(" (*specify either psrfbins or psrfdf!)"); - puts(" frequencies to be protected (def=2)"); + puts("-psrfdf* - Delta-frequency (on either side) of the fundamental & harmonic"); + puts(" frequencies to be protected (def=default psrfbins below)"); + puts("-psrfbins* - *Nbins (on either side) of the fundamental & harmonic"); + puts(" frequencies to be protected (def=8)"); + puts(" (*specify either psrfbins or psrfdf!)"); + puts(""); + puts("Add-on options:"); + puts("-n - specify output number of bits (def=input)"); + puts("-zeordm - remove 0-dm powers before writing out data"); + puts("-pcl - assume that input data may have some parts replaced"); + puts(" by 0s or mean/median by some other RFI excision program"); + puts("-T - number of time samples to avg before output (def=0)"); puts("-headerless - do not broadcast resulting header (def=broadcast)"); puts("-bst - Starting block number to be processed (def=1, i.e. start of file)"); puts("-nbl - No. of blocks to be processed (def=till EoF)"); puts(""); + puts("For GMRT data in native format:"); + puts("-gm - For gmrt raw-data, name of the file that contains header information"); + puts("-gmtstamp - For gmrt data, name of the file that contains time-stamp information"); + puts(""); + puts("Options to control which RFI excision methods to skip (if at all):"); + puts("-noRFIx - Do not excise any RFI "); + puts("-noFDx - Do not excise periodic RFI in the Fourier domain"); + puts("-noTx - Do not excise spiky RFI from channel/band-averaged time-series"); + puts("-noSx - Do not excise narrow-band RFI from spectra"); + puts("-noMSx - Do not excise narrow-band RFI using mean spectra"); + puts("-noVSx - Do not excise narrow-band RFI using variance spectra"); + puts("-noSclip - Do not clip above clip-threshold in individual spectra"); + puts(""); } int file_exists(char *filename) @@ -99,7 +117,7 @@ void print_version(char *program, char *argument) { if ( (strings_equal(argument,"-v")) || (strings_equal(argument,"--version"))) { - printf("PROGRAM: %s is part of rfiClean version: %.1f\n",program,RFICLEAN_VERSION); + printf("PROGRAM: %s is part of RFIClean version: %.1f\n",program,RFICLEAN_VERSION); exit(0); } } @@ -112,6 +130,7 @@ void main (int argc, char *argv[]) /* set up default global variables */ obits=headerless=naddt=nsamp=0; + RFIx=rfiFDx=rfiTx=rfiSx=rfiMSx=rfiVSx=rfiSclip=true; ibits=0; fthresh = 6.0; forcefthresh = 1000.0; @@ -146,6 +165,22 @@ void main (int argc, char *argv[]) if (strings_equal(argv[i],"-t")) { i++; naddt=atoi(argv[i]); + } else if (strings_equal(argv[i],"-noRFIx")) { + RFIx = false; + } else if (strings_equal(argv[i],"-noFDx")) { + rfiFDx = false; + } else if (strings_equal(argv[i],"-noTx")) { + rfiTx = false; + } else if (strings_equal(argv[i],"-noSx")) { + rfiSx = false; + rfiMSx = false; + rfiVSx = false; + } else if (strings_equal(argv[i],"-noMSx")) { + rfiMSx = false; + } else if (strings_equal(argv[i],"-noVSx")) { + rfiVSx = false; + } else if (strings_equal(argv[i],"-noSclip")) { + rfiSclip = false; } else if (strings_equal(argv[i],"-normf")) { fnorm=1; } else if (strings_equal(argv[i],"-normt")) { @@ -186,6 +221,9 @@ void main (int argc, char *argv[]) } else if (strings_equal(argv[i],"-st")) { i++; sthresh=atof(argv[i]); + } else if (strings_equal(argv[i],"-clt")) { + i++; + clipthresh=atof(argv[i]); } else if (strings_equal(argv[i],"-n")) { i++; obits=atoi(argv[i]); @@ -238,24 +276,25 @@ void main (int argc, char *argv[]) rficlean_help(); exit(0); } + if(!rfiVSx && !rfiMSx) rfiSx = false; // some sanity checks if (psrf<1000000.0 && fthresh<4.0 && forcefthresh>0) fthresh=4.0; if (psrfdf>0.0 && psrfbins > 0){ - printf ("Both psrfdf and psrfbins are specified!!"); - printf ("Using the delta-F information from psrfdf."); + printf ("\nBoth psrfdf and psrfbins are specified!!\n"); + printf ("Using the delta-F information from psrfdf.\n"); psrfbins = -1; } - if (psrfdf<0.0 && psrfbins<0){ - printf ("Nether psrfdf nor psrfbins is specified!!"); - printf ("Using psrfbins=8"); + if (psrf<100000.0 && psrfdf<0.0 && psrfbins<0){ + printf ("\nNether psrfdf nor psrfbins is specified!!\n"); + printf ("Using psrfbins=8\n"); psrfbins = 8; } /* read in the header to establish what the input data are... */ if (nifs>1){ - printf ("Cannot process data with IFs more than 1."); - printf ("Halting !!"); + printf ("Cannot process data with IFs more than 1.\n"); + printf ("Halting !!\n"); exit(0); } if (gm>0) { diff --git a/src/rficlean_data.c b/src/rficlean_data.c index 2f6d66d..94c16f4 100644 --- a/src/rficlean_data.c +++ b/src/rficlean_data.c @@ -152,7 +152,7 @@ void rficlean_data(FILE *input, FILE *output) printf (" All buffers prepared! \n\n"); /* main loop */ - printf (" Now rfiClean-ing (and 0-DM cleaning & downsampling, if asked for), and writing out the data...\n \n"); + printf (" Now RFIClean-ing (and 0-DM cleaning & downsampling, if asked for), and writing out the data...\n \n"); istart = 0; iblock = 0; while ((ns=read_block(input,nbits,fblock,nsblk,byte_offset))>0 && iblock Date: Tue, 9 Feb 2021 02:09:41 +0100 Subject: [PATCH 2/4] Improve command line documentation. --- src/rficlean.c | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/rficlean.c b/src/rficlean.c index 206697d..117ed62 100644 --- a/src/rficlean.c +++ b/src/rficlean.c @@ -42,48 +42,48 @@ int help_required(char *string) void rficlean_help() { puts(""); - puts("rficlean - clean some periodic and/or narrow-band and/or spiky/bursty RFIs in filterbank data\n"); + puts("rficlean - clean some periodic and/or narrow-band and/or spiky/bursty RFI from filterbank data\n"); puts("usage: rficlean -{options} {input-filename} \n"); puts(""); puts("General options:\n"); - puts(" - filterbank data file (def=stdin)"); - puts("-t - number of time samples in a block (def=4096)"); - puts("-ft - specify threshold for FT-cleaning (def=6.0)"); - puts("-white - whitten each spectrum before RFI excision"); + puts(" - Input data file (def=stdin)"); + puts("-t - Number of time samples in a block (def=4096)"); + puts("-ft - Threshold for FT-cleaning (def=6.0)"); + puts("-white - Whitten each spectrum before RFI excision"); puts("-nharm - Number of fundamental+harmonics to be excised"); - puts(" (def=1, i.e., only fundamental)"); - puts("-st - specify threshold for timeseries cleaning (def=10.0)"); - puts("-rt - specify threshold for channel cleaning (def=4.0)"); - puts("-clt - specify threshold for channel clipping (def=5.0)"); - puts("-ps - File-name of the output diagnostic plot (def=rficlean_output.ps)"); - puts("-o - specify output filename (def=stdout)"); + puts(" (def=1, i.e., only fundamental)"); + puts("-st - Threshold for timeseries cleaning (def=10.0)"); + puts("-rt - Threshold for channel cleaning (def=4.0)"); + puts("-clt - Threshold for channel clipping (def=5.0)"); + puts("-ps - Output diagnostic plot file-name (def=rficlean_output.ps)"); + puts("-o - Output filterbank filename (def=stdout)"); puts(""); puts("To safeguard a known periodic signal:"); - puts("-psrf - fundamental freq. (Hz) of pulsar to be protected (def=none)"); - puts("-psrfdf* - Delta-frequency (on either side) of the fundamental & harmonic"); - puts(" frequencies to be protected (def=default psrfbins below)"); - puts("-psrfbins* - *Nbins (on either side) of the fundamental & harmonic"); - puts(" frequencies to be protected (def=8)"); + puts("-psrf - Fundamental freq. (Hz) of pulsar to be protected (def=none)"); + puts("-psrfdf* - Delta-freq. (on either side) of the fundamental & harmonics"); + puts(" to be protected (def=default psrfbins below)"); + puts("-psrfbins* - *Nbins (on either side) of the fundamental & harmonics"); + puts(" to be protected (def=8)"); puts(" (*specify either psrfbins or psrfdf!)"); puts(""); puts("Add-on options:"); - puts("-n - specify output number of bits (def=input)"); - puts("-zeordm - remove 0-dm powers before writing out data"); - puts("-pcl - assume that input data may have some parts replaced"); - puts(" by 0s or mean/median by some other RFI excision program"); - puts("-T - number of time samples to avg before output (def=0)"); - puts("-headerless - do not broadcast resulting header (def=broadcast)"); - puts("-bst - Starting block number to be processed (def=1, i.e. start of file)"); + puts("-n - Output number of bits (def=input)"); + puts("-zeordm - Zero-DM filtering"); + puts("-pcl - Assume that input data may have some parts replaced"); + puts(" by 0s or mean/median by other RFI excision program"); + puts("-T - Number of time samples to avg before output (def=0)"); + puts("-headerless - Do not broadcast resulting header (def=broadcast)"); + puts("-bst - Starting block no. to be processed (def=1, =>start of file)"); puts("-nbl - No. of blocks to be processed (def=till EoF)"); puts(""); puts("For GMRT data in native format:"); - puts("-gm - For gmrt raw-data, name of the file that contains header information"); - puts("-gmtstamp - For gmrt data, name of the file that contains time-stamp information"); + puts("-gm - File that contains header information"); + puts("-gmtstamp - File that contains time-stamp information"); puts(""); puts("Options to control which RFI excision methods to skip (if at all):"); puts("-noRFIx - Do not excise any RFI "); puts("-noFDx - Do not excise periodic RFI in the Fourier domain"); - puts("-noTx - Do not excise spiky RFI from channel/band-averaged time-series"); + puts("-noTx - Do not excise spiky RFI from channel/band-avged timeseries"); puts("-noSx - Do not excise narrow-band RFI from spectra"); puts("-noMSx - Do not excise narrow-band RFI using mean spectra"); puts("-noVSx - Do not excise narrow-band RFI using variance spectra"); From 624b2a60a7befd713fe40e601b935aa6d907e624 Mon Sep 17 00:00:00 2001 From: Yogesh Maan Date: Tue, 9 Feb 2021 02:16:37 +0100 Subject: [PATCH 3/4] Include bash script for GMRT data --- .gitignore | 2 +- bin/crp_rficlean_gm.sh | 98 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) create mode 100755 bin/crp_rficlean_gm.sh diff --git a/.gitignore b/.gitignore index ca0781d..0624e08 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,7 @@ Mkfile.old dkms.conf ##bin/rficlean.flags # include this in the package as an example -bin/crp_rficlean_gm.sh +#bin/crp_rficlean_gm.sh bin/rficlean bin/read_n_make_plot src/read_n_make_plot.c diff --git a/bin/crp_rficlean_gm.sh b/bin/crp_rficlean_gm.sh new file mode 100755 index 0000000..4499f8a --- /dev/null +++ b/bin/crp_rficlean_gm.sh @@ -0,0 +1,98 @@ +#!/bin/bash +#======================================================================= +# Crude parallelization of RFIClean for processing GMRT data files. +# +# Yogesh Maan, Dec. 2019 +#======================================================================= +# + + + +#------------------------------------------------------------------------------ +if [ $# -lt 6 ] ; then + echo "========================================================================================" + echo " Usage: " + echo " crp_rficlean_gm.sh " + echo " " + echo "========================================================================================" + exit 0 +fi +outfile=$1 +flagfile=$2 +Np=$3 +infile=$4 +gminfo_file=$5 +extra_flag=$6 +#------------------------------------------------------------------------------ + +rtag=${outfile%*.fil} # tag for part-files + +##------------------------------- +## read flags from the flag-file +block=`sed -n 1p $flagfile` +flag=`sed -n 2p $flagfile` +#echo "block-size: $block" +#echo "flag: '$flag'" +## read nchan from the gm-info-file +nchans=`sed -n 4p $gminfo_file` +##------------------------------- + + +# deduce number of blocks to be processed in each part +fsize=$(stat -cL%s "$infile") +nsamples=`echo "scale=0; $fsize/(2*$nchans)" | bc` ## gmrt data --> 2 bytes/sample +Nb=`echo "scale=0; $nsamples/($Np*$block) + 1" | bc` +#echo "nsamples, Nblocks: $nsamples, $Nb, $Np, $block ---" + + +echo "Starting parallel-RFIClean processing at: " +date +echo "" + +##-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= +# prepare and execute the commands for all parts +mkdir -p RFIClean_ps +combine_cmd="cat " +remove_cmd="rm " +count=0 +for (( c=1; c<=$Np; c++ )); do + bst=$(($count*$Nb + 1)) + count=$((count+1)) + #echo $count + tempout=${rtag}_part${count}.fil + psfile=${rtag}_part${count}.ps + pdffile=${rtag}_part${count}.pdf + #------------------------------------- + if [ $count -eq 1 ]; then + cmd="/home/ymaan/bin/rficlean -t $block $flag -o $outfile -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb -gm $gminfo_file $extra_flag && ps2pdf RFIClean_ps/$psfile RFIClean_ps/$pdffile && rm RFIClean_ps/$psfile &" + elif [ $count -eq $Np ]; then + combine_cmd="${combine_cmd} ${tempout}" + remove_cmd="${remove_cmd} ${tempout}" + cmd="/home/ymaan/bin/rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -headerless -gm $gminfo_file $extra_flag && ps2pdf RFIClean_ps/$psfile RFIClean_ps/$pdffile && rm RFIClean_ps/$psfile &" + else + combine_cmd="${combine_cmd} ${tempout}" + remove_cmd="${remove_cmd} ${tempout}" + cmd="/home/ymaan/bin/rficlean -t $block $flag -o $tempout -ps RFIClean_ps/$psfile $infile -bst $bst -nbl $Nb -headerless -gm $gminfo_file $extra_flag && ps2pdf RFIClean_ps/$psfile RFIClean_ps/$pdffile && rm RFIClean_ps/$psfile &" + fi + echo $cmd + eval $cmd +done +wait +##-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= + + +# Now combine all parts and then remove the parts +combine_cmd="${combine_cmd} >> ${outfile}" +#echo $combine_cmd +#echo $remove_cmd +cmd="$combine_cmd && $remove_cmd" +echo $cmd +eval $cmd + + +echo "Finished parallel-RFIClean processing at: " +date +echo "" + +exit 0 + From 6abe2524a454ed50169947fde26ead31d4a0d60e Mon Sep 17 00:00:00 2001 From: Yogesh Maan Date: Tue, 9 Feb 2021 02:20:21 +0100 Subject: [PATCH 4/4] . --- COPYING | 12 +++++++----- COPYRIGHT | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/COPYING b/COPYING index 623b625..aa77660 100644 --- a/COPYING +++ b/COPYING @@ -55,7 +55,7 @@ patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. - + GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions: License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in @@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - + 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is @@ -225,7 +225,7 @@ impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - + 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License @@ -278,7 +278,7 @@ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS - + How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest @@ -338,3 +338,5 @@ proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. + + diff --git a/COPYRIGHT b/COPYRIGHT index a1ddc7c..ea9bda0 100644 --- a/COPYRIGHT +++ b/COPYRIGHT @@ -16,3 +16,4 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * */ +