From f54e295f5138a8822e7ede7dec0a266e0167af9d Mon Sep 17 00:00:00 2001 From: Gareth S Cabourn Davies Date: Tue, 10 Oct 2023 09:16:55 +0100 Subject: [PATCH] Make logging better for pycbc_make_offline_workflow (#4517) * Make logging better for pycbc_make_offline_workflow * comment * Fix bug * CC complaint - there are some other, but these havent been changed by this PR * CC * Update pycbc/workflow/core.py Co-authored-by: Tito Dal Canton * IH/TDC comments * use warnings module --------- Co-authored-by: Tito Dal Canton --- .../pycbc_make_offline_search_workflow | 11 +++++----- pycbc/workflow/configuration.py | 4 ++++ pycbc/workflow/core.py | 8 ++++++-- pycbc/workflow/pegasus_workflow.py | 20 ++++++++++++++----- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/bin/workflows/pycbc_make_offline_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow index bedbe22a452..9b3a7da8678 100755 --- a/bin/workflows/pycbc_make_offline_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -57,17 +57,18 @@ def ifo_combos(ifos): for ifocomb in combinations: yield ifocomb - -# Log to the screen until we know where the output file is -logging.basicConfig(format='%(asctime)s:%(levelname)s : %(message)s', - level=logging.INFO) - parser = argparse.ArgumentParser(description=__doc__[1:]) parser.add_argument('--version', action='version', version=__version__) +parser.add_argument('--verbose', action='count', + help="Incrementally add more verbosity") wf.add_workflow_command_line_group(parser) wf.add_workflow_settings_cli(parser) args = parser.parse_args() +# By default, we do logging.info, each --verbose adds a level of verbosity +logging_level = args.verbose + 1 if args.verbose else logging.INFO +pycbc.init_logging(logging_level) + container = wf.Workflow(args, args.workflow_name) workflow = wf.Workflow(args, args.workflow_name + '-main') finalize_workflow = wf.Workflow(args, args.workflow_name + '-finalization') diff --git a/pycbc/workflow/configuration.py b/pycbc/workflow/configuration.py index 0943e286fe3..e5c59c17de1 100644 --- a/pycbc/workflow/configuration.py +++ b/pycbc/workflow/configuration.py @@ -28,6 +28,7 @@ """ import os +import logging import stat import shutil import subprocess @@ -87,6 +88,9 @@ def resolve_url(url, directory=None, permissions=None, copy_to_cwd=True): # it needs to be available when documentation runs in the CI, and I # can't get it to install in the GitHub CI import ciecplib + # Make the scitokens logger a little quieter + # (it is called through ciecpclib) + logging.getLogger('scitokens').setLevel(logging.root.level + 10) with ciecplib.Session() as s: if u.netloc in ("git.ligo.org", "code.pycbc.phy.syr.edu"): # authenticate with git.ligo.org using callback diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index 3b03ae5d400..1c2aa7da6ad 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -224,8 +224,12 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None, # being directly accessible on the execution site. # CVMFS is perfect for this! As is singularity. self.exe_pfns[exe_site] = exe_path - logging.info("Using %s executable " - "at %s on site %s" % (name, exe_url.path, exe_site)) + logging.debug( + "Using %s executable at %s on site %s", + name, + exe_url.path, + exe_site + ) # FIXME: This hasn't yet been ported to pegasus5 and won't work. # Pegasus describes two ways to work with containers, and I need diff --git a/pycbc/workflow/pegasus_workflow.py b/pycbc/workflow/pegasus_workflow.py index b2fc880ced4..e442072e28a 100644 --- a/pycbc/workflow/pegasus_workflow.py +++ b/pycbc/workflow/pegasus_workflow.py @@ -28,8 +28,10 @@ """ import os import shutil +import logging import tempfile import subprocess +import warnings from packaging import version from urllib.request import pathname2url from urllib.parse import urljoin, urlsplit @@ -315,6 +317,13 @@ class Workflow(object): """ def __init__(self, name='my_workflow', directory=None, cache_file=None, dax_file_name=None): + # Pegasus logging is fairly verbose, quieten it down a bit + # This sets the logger to one level less verbose than the root + # (pycbc) logger + + # Get the logger associated with the Pegasus workflow import + pegasus_logger = logging.getLogger('Pegasus') + pegasus_logger.setLevel(logging.root.level + 10) self.name = name self._rc = dax.ReplicaCatalog() self._tc = dax.TransformationCatalog() @@ -340,7 +349,7 @@ def __init__(self, name='my_workflow', directory=None, cache_file=None, self.filename = dax_file_name self._adag = dax.Workflow(self.filename) - # A pegasus job version of this workflow for use if it isncluded + # A pegasus job version of this workflow for use if it is included # within a larger workflow self._as_job = SubWorkflow(self.filename, is_planned=False, _id=self.name) @@ -843,10 +852,11 @@ def insert_into_dax(self, rep_cat, sites): @classmethod def from_path(cls, path): """Takes a path and returns a File object with the path as the PFN.""" - logging.warn("The from_path method in pegasus_workflow is deprecated. " - "Please use File.from_path (for output files) in core.py " - "or resolve_url_to_file in core.py (for input files) " - "instead.") + warnings.warn("The from_path method in pegasus_workflow is " + "deprecated. Please use File.from_path (for " + "output files) in core.py or resolve_url_to_file " + "in core.py (for input files) instead.", + DeprecationWarning) urlparts = urlsplit(path) site = 'nonlocal' if (urlparts.scheme == '' or urlparts.scheme == 'file'):