Merge branch 'gwastro:master' into modular_stat

.github/PULL_REQUEST_TEMPLATE/standard_template.md

@@ -0,0 +1,52 @@
+<!---
+Please add a title which is a concise description of what you are doing,
+e.g. 'Fix bug with numpy import in pycbc_coinc_findtrigs' or 'add high frequency sky location dependent response for long detectors'
+--->
+
+<!---
+This is a brief template for making pull requests for PyCBC.
+This is _not_ a proscriptive template - you can use a different style if you want.
+Please do think about the questions posed here and whether the details will be useful to include in your PR
+Please add sufficient details so that people looking back at the request with no context around the work
+can understand the changes.
+To choose reviewers, please look at the git blame for the code you are changing (if applicable),
+or discuss in the gwastro slack.
+Please add labels as appropriate
+-->
+
+- [ ] The author of this pull request confirms they will adhere to the [code of conduct](https://github.com/gwastro/pycbc/blob/master/CODE_OF_CONDUCT.md)
+
+## Standard information about the request
+
+<!--- Some basic info about the change (delete as appropriate) --->
+This is a: bug fix, new feature, efficiency update, other (please describe)
+
+<!--- What codes will this affect? (delete as apropriate)
+If you do not know which areas will be affected, please ask in the gwastro #pycbc-code slack
+--->
+This change affects: the offline search, the live search, inference, PyGRB
+
+<!--- What code areas will this affect? (delete as apropriate) --->
+This change changes: documentation, result presentation / plotting, scientific output
+
+<!--- Some things which help with code management (delete as appropriate) --->
+This change: has appropriate unit tests, follows style guidelines (See e.g. [PEP8](https://peps.python.org/pep-0008/)), has been proposed using the [contribution guidelines](https://github.com/gwastro/pycbc/blob/master/CONTRIBUTING.md)
+
+<!--- Notes about the effect of this change --->
+This change will: break current functionality, require additional dependencies, require a new release, other (please describe)
+
+## Motivation
+<!--- Describe why your changes are being made -->
+
+## Contents
+<!--- Describe your changes, this doesn't need to be a line-by-line code change discussion,
+but rather a general discussion of the methods chosen -->
+
+## Links to any issues or associated PRs
+<!--- If this is fixing / working around an already-reported issue, please link to it here --->
+
+## Testing performed
+<!--- Describe tests for the code changes, either already performed or to be performed -->
+
+## Additional notes
+<!--- Anything which does not fit in the above sections -->

.github/workflows/inference-workflow.yml

@@ -25,7 +25,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |

.github/workflows/search-workflow.yml

@@ -30,7 +30,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |

.github/workflows/tmpltbank-workflow.yml

@@ -29,7 +29,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |

.github/workflows/workflow-tests.yml

@@ -34,7 +34,7 @@ jobs:
         wget -qO - https://download.pegasus.isi.edu/pegasus/gpg.txt | sudo apt-key add -
         echo "deb https://download.pegasus.isi.edu/pegasus/ubuntu bionic main" | sudo tee -a /etc/apt/sources.list
         sudo apt-get -o Acquire::Retries=3 update
-        sudo apt-get -o Acquire::Retries=3 install pegasus
+        sudo apt-get -o Acquire::Retries=3 install pegasus=5.0.6-1+ubuntu18
     - run: sudo apt-get -o Acquire::Retries=3 install *fftw3* intel-mkl*
     - name: Install pycbc
       run: |

bin/all_sky_search/pycbc_bin_trigger_rates_dq

@@ -46,6 +46,18 @@ logging.info('Start')
 
 ifo, flag_name = args.flag_name.split(':')
 
+if args.gating_windows:
+    gate_times = []
+    with h5.File(args.trig_file, 'r') as trig_file:
+        logging.info('Getting gated times')
+        try:
+            gating_types = trig_file[f'{ifo}/gating'].keys()
+            for gt in gating_types:
+                gate_times += list(trig_file[f'{ifo}/gating/{gt}/time'][:])
+            gate_times = np.unique(gate_times)
+        except KeyError:
+            logging.warning('No gating found in trigger file')
+
 trigs = SingleDetTriggers(
     args.trig_file,
     ifo,
@@ -134,6 +146,7 @@ with h5.File(args.output_file, 'w') as f:
             frac_dt = abs(segs) / livetime
             dq_rates[state] = frac_eff / frac_dt
         bin_grp['dq_rates'] = dq_rates
+        bin_grp['num_triggers'] = len(trig_times_bin)
 
     # save dq state segments
     for dq_state, segs in dq_state_segs_dict.items():
@@ -142,7 +155,10 @@ with h5.File(args.output_file, 'w') as f:
         starts, ends = segments_to_start_end(segs)
         dq_grp['segment_starts'] = starts
         dq_grp['segment_ends'] = ends
+        dq_grp['livetime'] = abs(segs)
 
     f.attrs['stat'] = f'{ifo}-dq_stat_info'
+    f.attrs['sngl_ranking'] = args.sngl_ranking
+    f.attrs['sngl_ranking_threshold'] = args.stat_threshold
 
 logging.info('Done!')

bin/all_sky_search/pycbc_fit_sngls_over_multiparam

@@ -174,18 +174,14 @@ parser = argparse.ArgumentParser(usage="",
 
 pycbc.add_common_pycbc_options(parser)
 parser.add_argument("--version", action=pycbc.version.Version)
-parser.add_argument("--template-fit-file",
+parser.add_argument("--template-fit-file", required=True,
                     help="hdf5 file containing fit coefficients for each"
                          " individual template. Required")
-parser.add_argument("--bank-file", default=None,
-                    help="hdf file containing template parameters. Required "
-                         "unless reading param from template fit file")
+parser.add_argument("--bank-file", required=True,
+                    help="hdf file containing template parameters. Required")
 parser.add_argument("--output", required=True,
                     help="Location for output file containing smoothed fit "
-                         "coefficients.  Required")
-parser.add_argument("--use-template-fit-param", action="store_true",
-                    help="Use parameter values stored in the template fit "
-                         "file as template_param for smoothing.")
+                         "coefficients. Required")
 parser.add_argument("--fit-param", nargs='+',
                     help="Parameter(s) over which to regress the background "
                          "fit coefficients. Required. Either read from "
@@ -196,20 +192,19 @@ parser.add_argument("--fit-param", nargs='+',
                          "multiple parameters, provide them as a list.")
 parser.add_argument("--approximant", default="SEOBNRv4",
                     help="Approximant for template duration. Default SEOBNRv4")
-parser.add_argument("--f-lower", type=float, default=0.,
-                    help="Starting frequency for calculating template "
-                         "duration, if not reading from the template fit file")
+parser.add_argument("--f-lower", type=float,
+                    help="Start frequency for calculating template duration.")
 parser.add_argument("--min-duration", type=float, default=0.,
                     help="Fudge factor for templates with tiny or negative "
                          "values of template_duration: add to duration values"
                          " before fitting. Units seconds.")
 parser.add_argument("--log-param", nargs='+',
-                    help="Take the log of the fit param before smoothing.")
+                    help="Take the log of the fit param before smoothing. "
+                         "Must be a list corresponding to fit params.")
 parser.add_argument("--smoothing-width", type=float, nargs='+', required=True,
-                    help="Distance in the space of fit param values (or the "
-                         "logs of them) to smooth over. Required. "
-                         "This must be a list corresponding to the smoothing "
-                         "parameters.")
+                    help="Distance in the space of fit param values (or their"
+                         " logs) to smooth over. Required. Must be a list "
+                         "corresponding to fit params.")
 parser.add_argument("--smoothing-method", default="smooth_tophat",
                     choices = _smooth_dist_func.keys(),
                     help="Method used to smooth the fit parameters; "
@@ -220,15 +215,14 @@ parser.add_argument("--smoothing-method", default="smooth_tophat",
                          "the smoothing until 500 triggers are reached. "
                          "'distance_weighted' weights the closest templates "
                          "with a normal distribution of width smoothing-width "
-                         "trucated at three smoothing-widths.")
+                         "truncated at three smoothing-widths.")
 parser.add_argument("--smoothing-keywords", nargs='*',
                     help="Keywords for the smoothing function, supplied "
                          "as key:value pairs, e.g. total_trigs:500 to define "
-                         "the number of templates in the n_closest smoothing "
-                         "method")
+                         "the number of templates for n_closest smoothing.")
 parser.add_argument("--output-fits-by-template", action='store_true',
                     help="If given, will output the input file fits to "
-                         "fit_by_template group")
+                         "fit_by_template group.")
 args = parser.parse_args()
 
 if args.smoothing_keywords:

bin/all_sky_search/pycbc_prepare_xml_for_gracedb

@@ -96,7 +96,7 @@ def read_psds(psd_files):
         psd = [group["psds"][str(i)] for i in range(len(group["psds"].keys()))]
         psds[ifo] = segmentlist(psd_segment(*segargs) for segargs in zip(
             psd, group["start_time"], group["end_time"]))
-        return psds
+    return psds
 
 psds = read_psds(args.psd_files)
 

bin/bank/pycbc_aligned_bank_cat

@@ -48,9 +48,8 @@ __program__ = "pycbc_aligned_bank_cat"
 parser = argparse.ArgumentParser(description=__doc__,
            formatter_class=tmpltbank.IndentedHelpFormatterWithNL)
 
+pycbc.add_common_pycbc_options(parser)
 parser.add_argument("--version", action="version", version=__version__)
-parser.add_argument("--verbose", action="store_true", default=False,
-                    help="verbose output")
 parser.add_argument("-i", "--input-glob",
                     help="file glob the list of paramters")
 parser.add_argument("-I", "--input-files", nargs='+',
@@ -76,11 +75,7 @@ tmpltbank.insert_base_bank_options(parser, match_req=False)
 
 options = parser.parse_args()
 
-if options.verbose:
-    log_level = logging.DEBUG
-else:
-    log_level = logging.WARN
-logging.basicConfig(format='%(asctime)s %(message)s', level=log_level)
+pycbc.init_logging(options.verbose)
 
 # Sanity check options
 if not options.output_file:

bin/bank/pycbc_aligned_stoch_bank

@@ -23,6 +23,7 @@ Stochastic aligned spin bank generator.
 import argparse
 import numpy
 import logging
+
 import pycbc
 import pycbc.version
 from pycbc import tmpltbank
@@ -45,8 +46,7 @@ parser = argparse.ArgumentParser(description=_desc,
 
 # Begin with code specific options
 parser.add_argument("--version", action="version", version=__version__)
-parser.add_argument("--verbose", action="store_true", default=False,
-                    help="verbose output")
+pycbc.add_common_pycbc_options(parser)
 parser.add_argument("-V", "--vary-fupper", action="store_true", default=False,
                     help="Use a variable upper frequency cutoff in laying "
                     "out the bank.  OPTIONAL.")
@@ -96,12 +96,7 @@ tmpltbank.insert_ethinca_metric_options(parser)
 
 opts = parser.parse_args()
 
-if opts.verbose:
-    log_level = logging.DEBUG
-else:
-    log_level = logging.WARN
-log_format='%(asctime)s %(message)s'
-logging.basicConfig(format=log_format, level=log_level)
+pycbc.init_logging(opts.verbose)
 
 # delete defaults for redundant options if not varying fupper
 if not opts.vary_fupper:

bin/bank/pycbc_bank_verification

@@ -26,14 +26,16 @@ import argparse
 import logging
 import numpy
 import h5py
+import matplotlib
+matplotlib.use('Agg')
+import pylab
+
 from ligo.lw import lsctables, utils as ligolw_utils
+
 import pycbc
 import pycbc.version
 from pycbc import tmpltbank, psd, strain
 from pycbc.io.ligolw import LIGOLWContentHandler
-import matplotlib
-matplotlib.use('Agg')
-import pylab
 
 
 __author__  = "Ian Harry <[email protected]>"
@@ -48,8 +50,7 @@ parser = argparse.ArgumentParser(description=__doc__,
 
 # Begin with code specific options
 parser.add_argument("--version", action="version", version=__version__)
-parser.add_argument("--verbose", action="store_true", default=False,
-                    help="verbose output")
+pycbc.add_common_pycbc_options(parser)
 parser.add_argument("--histogram-output-file", action="store", default=None,
                     help="Output a histogram of fitting factors to the "
                     "supplied file. If not given no histogram is produced.")
@@ -111,11 +112,7 @@ pycbc.strain.insert_strain_option_group(parser)
 
 opts = parser.parse_args()
 
-if opts.verbose:
-    log_level = logging.DEBUG
-else:
-    log_level = logging.WARN
-logging.basicConfig(format='%(asctime)s %(message)s', level=log_level)
+pycbc.init_logging(opts.verbose)
 
 # Sanity check options
 tmpltbank.verify_metric_calculation_options(opts, parser)

bin/bank/pycbc_brute_bank

@@ -19,16 +19,21 @@
 
 """Generate a bank of templates using a brute force stochastic method.
 """
-import numpy, h5py, logging, argparse, numpy.random
+import numpy
+import h5py
+import logging
+import argparse
+import numpy.random
+from scipy.stats import gaussian_kde
+
 import pycbc.waveform, pycbc.filter, pycbc.types, pycbc.psd, pycbc.fft, pycbc.conversions
 from pycbc import transforms
 from pycbc.waveform.spa_tmplt import spa_length_in_time
 from pycbc.distributions import read_params_from_config
 from pycbc.distributions.utils import draw_samples_from_config, prior_from_config
-from scipy.stats import gaussian_kde
 
 parser = argparse.ArgumentParser(description=__doc__)
-parser.add_argument('--verbose', action='store_true')
+pycbc.add_common_pycbc_options(parser)
 parser.add_argument('--output-file', required=True,
     help='Output file name for template bank.')
 parser.add_argument('--input-file',
@@ -71,10 +76,10 @@ parser.add_argument('--tau0-end', type=float)
 parser.add_argument('--tau0-cutoff-frequency', type=float, default=15.0)
 pycbc.psd.insert_psd_option_group(parser)
 args = parser.parse_args()
+
 pycbc.init_logging(args.verbose)
 numpy.random.seed(args.seed)
 
-
 # Read the .ini file if it's in the input.
 if args.input_config is not None:
     config_parser = pycbc.types.config.InterpolatingConfigParser()

bin/bank/pycbc_coinc_bank2hdf

@@ -23,6 +23,7 @@ with their template.
 import argparse
 import logging
 import numpy
+
 import pycbc
 from pycbc.waveform import bank
 
@@ -56,6 +57,7 @@ def parse_parameters(parameters):
 parser = argparse.ArgumentParser()
 parser.add_argument('--version', action='version',
                     version=pycbc.version.git_verbose_msg)
+pycbc.add_common_pycbc_options(parser)
 parser.add_argument('--bank-file', required=True,
                     help="The bank file to load. Must end in '.xml[.gz]' "
                          "and must contain a SnglInspiral table or must end "
@@ -82,8 +84,6 @@ bank.add_approximant_arg(parser,
 parser.add_argument("--force", action="store_true", default=False,
                     help="Overwrite the given hdf file if it exists. "
                          "Otherwise, an error is raised.")
-parser.add_argument("--verbose", action="store_true", default=False,
-                    help="Be verbose.")
 args = parser.parse_args()
 
 pycbc.init_logging(args.verbose)

bin/bank/pycbc_geom_aligned_2dstack

@@ -31,6 +31,7 @@ import copy
 import numpy
 import logging
 import h5py
+
 import pycbc.tmpltbank
 import pycbc.version
 from pycbc import pnutils
@@ -47,10 +48,9 @@ usage = """usage: %prog [options]"""
 _desc = __doc__[1:]
 parser = argparse.ArgumentParser(usage, description=_desc,
            formatter_class=pycbc.tmpltbank.IndentedHelpFormatterWithNL)
+pycbc.add_common_pycbc_options(parser)
 # Code specific options
 parser.add_argument('--version', action='version', version=__version__)
-parser.add_argument("--verbose", action="store_true", default=False,\
-                    help="verbose output")
 parser.add_argument("--pn-order", action="store", type=str,\
                   default=None,\
                   help="Determines the PN order to use. Note that if you "+\
@@ -106,11 +106,7 @@ opts = parser.parse_args()
 opts.eval_vec4_depth = not opts.skip_vec4_depth
 opts.eval_vec5_depth = not opts.skip_vec5_depth
 
-if opts.verbose:
-    log_level = logging.DEBUG
-else:
-    log_level = logging.WARN
-logging.basicConfig(format='%(asctime)s %(message)s', level=log_level)
+pycbc.init_logging(opts.verbose)
 
 # Sanity check options
 if not opts.pn_order: