Preparing for veto file handling in PyGRB (gwastro#4929)

* Missing empty line

* grb_utils.py preparation to handle veto-files and segment files

* pycbc_make_offline_grb_workflow update based on grb_utils.py upgrade

* Updated hacky pygrb stuff in jobsetup.py

* First batch of updates to handle the VDF in PyGRB and streamline pygrb_postprocessing_utils.py

* Removing build_veto_filelist from pycbc_pygrb_minifollowups

* Better variable naming and string search in jobsetup.py

* Removing redundant variables

* Corrected call to segments plots

* Better argparse syntax

* No f-string in logging

* assert instead of if

bin/pycbc_multi_inspiral

@@ -74,6 +74,7 @@ def slide_limiter(args):
         num_slides = 1
     return num_slides
 
+
 # The following block of lines sets up the command-line interface (CLI) for the
 # pycbc_multi_inspiral executable.
 time_init = time.time()

bin/pygrb/pycbc_make_offline_grb_workflow

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-# Copyright (C) 2015 Andrew R. Williamson
+# Copyright (C) 2015 Andrew R. Williamson, Francesco Pannarale
 #
 # This program is free software; you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the
@@ -79,7 +79,6 @@ sciSegsFile = _workflow.get_segments_file(wflow,
                                           'science',
                                           'segments-science',
                                           seg_dir)
-
 sciSegs = {}
 for ifo in wflow.ifos:
     sciSegs[ifo] = sciSegsFile.segment_dict[ifo+':science']
@@ -130,14 +129,15 @@ else:
 
 sciSegs = segmentlistdict(sciSegs)
 if onSrc is None:
-    plot_met = make_grb_segments_plot(wflow, sciSegs, triggertime, triggername,
-                                      seg_dir, fail_criterion=offSrc)
+    plot_met = make_grb_segments_plot(
+        wflow, sciSegs, triggertime, triggername,
+        seg_dir, fail_criterion=offSrc)
     logging.info("Making segment plot and exiting.")
     sys.exit()
 else:
     plot_met = make_grb_segments_plot(
-            wflow, sciSegs, triggertime, triggername, seg_dir,
-            coherent_seg=offSrc[tuple(offSrc.keys())[0]][0])
+        wflow, sciSegs, triggertime, triggername, seg_dir,
+        coherent_seg=offSrc[tuple(offSrc.keys())[0]][0])
     segs_plot = _workflow.File(plot_met[0], plot_met[1], plot_met[2],
                                file_url=plot_met[3])
     segs_plot.add_pfn(segs_plot.cache_entry.path, site="local")
@@ -193,10 +193,12 @@ if wflow.cp.has_option("workflow-condition_strain", "do-gating"):
                                            int(sciSegs[ifo][0][1]))
 
 ifos = sorted(sciSegs.keys())
-ifo = ifos[0]
 wflow.ifos = ifos
 datafind_veto_files = _workflow.FileList([])
 
+# Convenience variable to operate on sciSegs within this executable
+ifo = ifos[0]
+
 # GATING
 if wflow.cp.has_option("workflow-condition_strain", "do-gating"):
     logging.info("Creating gating jobs.")
@@ -234,12 +236,17 @@ if wflow.cp.has_option("workflow-condition_strain", "do-gating"):
         gated_frames.convert_to_lal_cache().tofile(
                             open(gated_cache.storage_path, "w"))
         datafind_files.append(gated_cache)
-
 datafind_veto_files.extend(datafind_files)
-ifo_list = sorted(sciSegs.keys())
-ifo = ifo_list[0]
-ifos = ''.join(ifo_list)
-wflow.ifos = ifos
+
+# Retrieve vetoes
+veto_file = None
+if wflow.cp.has_option("workflow-segments", "segments-vetoes"):
+    veto_file = _workflow.get_segments_file(wflow,
+                                            'vetoes', 
+                                            'segments-vetoes',
+                                            seg_dir,
+                                            tags=['veto'])
+    datafind_veto_files.append(veto_file)
 
 # Config file consistency check for IPN GRBs
 if wflow.cp.has_option("workflow-inspiral", "ipn-search-points") \
@@ -262,6 +269,7 @@ else:
 if wflow.cp.has_option('workflow-inspiral', 'bank-veto-bank-file'):
     bank_veto_file = configparser_value_to_file(wflow.cp, 'workflow-inspiral',
                                                 'bank-veto-bank-file')
+    bank_veto_file.description += '_BANK_VETO_BANK'
     bank_veto_file = _workflow.FileList([bank_veto_file])
     datafind_veto_files.extend(bank_veto_file)
 
@@ -453,7 +461,8 @@ if post_proc_method == "PYGRB_OFFLINE":
                                                                clustered_files,
                                                                inj_find_files,
                                                                full_bank_file,
-                                                               seg_dir)
+                                                               seg_dir,
+                                                               veto_file=veto_file)
         logging.info('Leaving results module')
 
 all_files.extend(pp_files)

bin/pygrb/pycbc_pygrb_minifollowups

@@ -36,7 +36,6 @@ import pycbc.events
 import pycbc.results.pygrb_postprocessing_utils as ppu
 from pycbc.results import layout
 from pycbc.workflow.plotting import PlotExecutable
-from pycbc.workflow.grb_utils import build_veto_filelist
 from pycbc.io.hdf import HFile
 
 __author__ = "Francesco Pannarale <[email protected]>"
@@ -72,10 +71,6 @@ def make_timeseries_plot(workflow, trig_file, snr_type, central_time,
                           ifos=workflow.ifos, out_dir=out_dir,
                           tags=tags+extra_tags).create_node()
     node.add_input_opt('--trig-file', trig_file)
-    # Pass the veto files
-    if workflow.cp.has_option('workflow', 'veto-files'):
-        veto_files = build_veto_filelist(workflow)
-        node.add_input_list_opt('--veto-files', veto_files)
     node.new_output_file_opt(workflow.analysis_time, '.png',
                              '--output-file', tags=extra_tags)
     # Quantity to be displayed on the y-axis of the plot
@@ -112,9 +107,6 @@ parser.add_argument('--followups-file',
                     help="HDF file with the triggers/injections to follow up")
 parser.add_argument('--wiki-file',
                     help="Name of file to save wiki-formatted table in")
-parser.add_argument('--veto-files', nargs='+', action="store",
-                    default=None, help="The location of the CATX veto " +
-                    "files provided as a list of space-separated values.")
 wf.add_workflow_command_line_group(parser)
 wf.add_workflow_settings_cli(parser, include_subdax_opts=True)
 ppu.pygrb_add_bestnr_cut_opt(parser)
@@ -143,8 +135,10 @@ num_events = int(workflow.cp.get_opt_tags('workflow-minifollowups',
                                           'num-events', ''))
 num_events = min(num_events, len(fp['BestNR'][:]))
 
-# Determine ifos used in the analysis
+# File instance of the input trigger file
 trig_file = resolve_url_to_file(os.path.abspath(args.trig_file))
+
+# Determine ifos used in the analysis
 ifos = ppu.extract_ifos(os.path.abspath(args.trig_file))
 num_ifos = len(ifos)