Skip to content

Commit

Permalink
refactor getCentralConfig Fix #7758 (#7792)
Browse files Browse the repository at this point in the history 
* refactor getCentralConfig Fix #7758

* pylint AND RESTORE CACHING TIME TO 30min !

* pep8
  • Loading branch information
@belforte
belforte authored Aug 8, 2023
1 parent fda4b6a commit 105991a
Showing 1 changed file with 79 additions and 74 deletions.
153 changes: 79 additions & 74 deletions src/python/CRABInterface/Utilities.py
View file
Original file line number Diff line number Diff line change
@@ -1,66 +1,61 @@
from __future__ import division
from __future__ import print_function
import logging

"""
The module contains some utility functions used by the various modules of the CRAB REST interface
"""
import os
from contextlib import contextmanager, nullcontext
from contextlib import contextmanager
from collections import namedtuple
from time import mktime, gmtime
import re
from hashlib import sha1
import cherrypy
import pycurl
import time
import subprocess
import io
import json
import copy
import pycurl
import cherrypy

from WMCore.WMFactory import WMFactory
from WMCore.REST.Error import ExecutionError, InvalidParameter
from WMCore.Services.CRIC.CRIC import CRIC
from WMCore.Services.pycurl_manager import ResponseHeader
from WMCore.REST.Server import RESTArgs

from Utils.Utilities import encodeUnicodeToBytes

from CRABInterface.Regexps import RX_CERT
"""
The module contains some utility functions used by the various modules of the CRAB REST interface
"""

CMSSitesCache = namedtuple("CMSSitesCache", ["cachetime", "sites"])
ConfigCache = namedtuple("ConfigCache", ["cachetime", "centralconfig"])

#These parameters are set in the globalinit (called in RESTBaseAPI)
# These parameters are set in the globalinit (called in RESTBaseAPI)
credServerPath = None


def getDBinstance(config, namespace, name):
if config.backend.lower() == 'mysql':
backend = 'MySQL'
elif config.backend.lower() == 'oracle':
backend = 'Oracle'

#factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name=name, namespace='Databases.%s.%s.%s' % (namespace, backend, name))
# factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name=name, namespace=f"Databases.{namespace}.{backend}.{name}")

return factory.loadObject(name)

return factory.loadObject( name )

def globalinit(credpath):
global credServerPath # pylint: disable=global-statement
credServerPath = credpath


def execute_command(command, logger, timeout):
"""
_execute_command_
Funtion to manage commands.
"""
import subprocess
import time

stdout, stderr, rc = None, None, 99999
proc = subprocess.Popen(
command, shell=True, cwd=os.environ['PWD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
command, shell=True, cwd=os.environ['PWD'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)

t_beginning = time.time()
Expand All @@ -71,29 +66,40 @@ def execute_command(command, logger, timeout):
seconds_passed = time.time() - t_beginning
if timeout and seconds_passed > timeout:
proc.terminate()
logger.error('Timeout in %s execution.' % command )
logger.error(f"Timeout in {command} execution.")
return stdout, rc

time.sleep(0.1)

stdout, stderr = proc.communicate()
rc = proc.returncode

logger.debug('Executing : \n command : %s\n output : %s\n error: %s\n retcode : %s' % (command, stdout, stderr, rc))
logger.debug(f"Executing : \n command : {command}\n output : {stdout}\n error: {stderr}\n retcode : {rc}")

return stdout, rc


#This is used in case git is down for more than 30 minutes
centralCfgFallback = None
# This is used in case git is down for more than 30 minutes
# NOTE BY STEFANO B: I do not like this global and the hacky way to
# have a fallback here. IMHO there should be a class which we instantiate
# for each configuration file that we want to retreive, with the fallback as
# a private variable. No need to expose cfgFallback to the outside.
# But at this time I am happy to have something which worke even if ugly.
# Another much needed cleanup would be to avoid using the same extConfCommon
# name for the structure loaded from gitlab and the differently formatted one
# done below, avoid the deletion of dictionary keys etc.
# But we can't spend all time cleaning up old code
#
global cfgFallback # pylint: disable=global-statement, global-at-module-level
cfgFallback = {}


def getCentralConfig(extconfigurl, mode):
"""Utility to retrieve the central configuration to be used for dynamic variables
arg str extconfigurl: the url pointing to the exteranl configuration parameter
arg str mode: also known as the variant of the rest (prod, preprod, dev, private)
return: the dictionary containing the external configuration for the selected mode."""

global centralCfgFallback # pylint: disable=global-statement

def retrieveConfig(externalLink):

hbuf = io.BytesIO()
Expand All @@ -109,53 +115,48 @@ def retrieveConfig(externalLink):

header = ResponseHeader(hbuf.getvalue())
if (header.status < 200 or header.status >= 300):
msg = "Reading %s returned %s." % (externalLink, header.status)
if centralCfgFallback:
msg = f"Reading {externalLink} returned {header.status}."
if externalLink in cfgFallback:
msg += "\nUsing cached values for external configuration."
cherrypy.log(msg)
return centralCfgFallback
else:
cherrypy.log(msg)
raise ExecutionError("Internal issue when retrieving external configuration from %s" % externalLink)
jsonConfig = bbuf.getvalue()

return jsonConfig
return cfgFallback[externalLink]
cherrypy.log(msg)
raise ExecutionError(f"Internal issue when retrieving configuration from {externalLink}")
try:
jsonConfig = bbuf.getvalue()
cfgFallback[externalLink] = jsonConfig
return jsonConfig
except ValueError:
return cfgFallback[externalLink]

extConfCommon = json.loads(retrieveConfig(extconfigurl))

# below 'if' condition is only added for the transition period from the old config file to the new one. It should be removed after some time.
if 'modes' in extConfCommon:
extConfSchedds = json.loads(retrieveConfig(extConfCommon['htcondorScheddsLink']))

# The code below constructs dict from below provided JSON structure
# { u'htcondorPool': '', u'compatible-version': [''], u'htcondorScheddsLink': '',
# u'modes': [{
# u'mode': '', u'backend-urls': {
# u'htcondorSchedds': [''], u'cacheSSL': '', u'baseURL': ''}}],
# u'banned-out-destinations': [], u'delegate-dn': ['']}
# to match expected dict structure which is:
# { u'compatible-version': [''], u'htcondorScheddsLink': '',
# 'backend-urls': {
# u'htcondorSchedds': {u'[email protected]': {u'proxiedurl': '', u'weightfactor': 1}},
# u'cacheSSL': '', u'baseURL': '', 'htcondorPool': ''},
# u'banned-out-destinations': [], u'delegate-dn': ['']}
extConfCommon['backend-urls'] = next((item['backend-urls'] for item in extConfCommon['modes'] if item['mode'] == mode), None)
extConfCommon['backend-urls']['htcondorPool'] = extConfCommon.pop('htcondorPool')
del extConfCommon['modes']

# if htcondorSchedds": [] is not empty, it gets populated with the specified list of schedds,
# otherwise it takes default list of schedds
if extConfCommon['backend-urls']['htcondorSchedds']:
extConfCommon['backend-urls']['htcondorSchedds'] = {k: v for k, v in extConfSchedds.items() if
k in extConfCommon['backend-urls']['htcondorSchedds']}
else:
extConfCommon["backend-urls"]["htcondorSchedds"] = extConfSchedds
centralCfgFallback = extConfCommon
extConfSchedds = json.loads(retrieveConfig(extConfCommon['htcondorScheddsLink']))

# The code below constructs dict from below provided JSON structure
# { u'htcondorPool': '', u'compatible-version': [''], u'htcondorScheddsLink': '',
# u'modes': [{
# u'mode': '', u'backend-urls': {
# u'htcondorSchedds': [''], u'cacheSSL': '', u'baseURL': ''}}],
# u'banned-out-destinations': [], u'delegate-dn': ['']}
# to match expected dict structure which is:
# { u'compatible-version': [''], u'htcondorScheddsLink': '',
# 'backend-urls': {
# u'htcondorSchedds': {u'[email protected]': {u'proxiedurl': '', u'weightfactor': 1}},
# u'cacheSSL': '', u'baseURL': '', 'htcondorPool': ''},
# u'banned-out-destinations': [], u'delegate-dn': ['']}
extConfCommon['backend-urls'] = next((item['backend-urls'] for item in extConfCommon['modes'] if item['mode'] == mode), None)
extConfCommon['backend-urls']['htcondorPool'] = extConfCommon.pop('htcondorPool')
del extConfCommon['modes']

# if htcondorSchedds": [] is not empty, it gets populated with the specified list of schedds,
# otherwise it takes default list of schedds
if extConfCommon['backend-urls']['htcondorSchedds']:
extConfCommon['backend-urls']['htcondorSchedds'] = {k: v for k, v in extConfSchedds.items() if
k in extConfCommon['backend-urls']['htcondorSchedds']}
else:
extConfCommon["backend-urls"]["htcondorSchedds"] = extConfSchedds
centralCfgFallback = extConfCommon

return centralCfgFallback
return extConfCommon


def conn_handler(services):
Expand All @@ -168,11 +169,15 @@ def conn_handler(services):
"""
def wrap(func):
def wrapped_func(*args, **kwargs):
if 'cric' in services and (not args[0].allCMSNames.sites or (args[0].allCMSNames.cachetime+1800 < mktime(gmtime()))):
if 'cric' in services and (not args[0].allCMSNames.sites or \
(args[0].allCMSNames.cachetime + 1800 < mktime(gmtime()))):
args[0].allCMSNames = CMSSitesCache(sites=CRIC().getAllPSNs(), cachetime=mktime(gmtime()))
args[0].allPNNNames = CMSSitesCache(sites=CRIC().getAllPhEDExNodeNames(), cachetime=mktime(gmtime()))
if 'centralconfig' in services and (not args[0].centralcfg.centralconfig or (args[0].centralcfg.cachetime+1800 < mktime(gmtime()))):
args[0].centralcfg = ConfigCache(centralconfig=getCentralConfig(extconfigurl=args[0].config.extconfigurl, mode=args[0].config.mode), cachetime=mktime(gmtime()))
if ('centralconfig' in services and \
(not args[0].centralcfg.centralconfig or (args[0].centralcfg.cachetime + 1800 < mktime(gmtime())))):
args[0].centralcfg = ConfigCache(
centralconfig=getCentralConfig(extconfigurl=args[0].config.extconfigurl, mode=args[0].config.mode),
cachetime=mktime(gmtime()))
return func(*args, **kwargs)
return wrapped_func
return wrap
Expand Down

0 comments on commit 105991a

Please sign in to comment.