improve HTTP header parsing (#5111)

dmwm · Nov 12, 2021 · 0fef759 · 0fef759
1 parent 8cb30af
commit 0fef759
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 7 deletions.
diff --git a/bin/crab.py b/bin/crab.py
@@ -149,11 +149,11 @@ def log_exception(exc_type, exc_value, tback):
         client()
         exitcode = 0 #no exceptions no errors
     except RESTInterfaceException as err:
+        exitcode=err.exitcode
         client.logger.info("The server answered with an error.")
         client.logger.debug("")
         err = str(err)
-        exitcode = int(re.findall(r'(?<=\<\sHTTP/1.1\s)[^\n\s]*', err)[-1])
-        if exitcode==503 and ("CMSWEB Error: Service unavailable") in err:
+        if ("CMSWEB Error: Service unavailable") in err:
             client.logger.info(schedInterv)
         if 'X-Error-Detail' in err:
             errorDetail = re.search(r'(?<=X-Error-Detail:\s)[^\n]*', err).group(0)

diff --git a/src/python/CRABClient/CrabRestInterface.py b/src/python/CRABClient/CrabRestInterface.py
@@ -56,6 +56,32 @@ def retriableError(http_code, curlExitCode):
     return retry
 
 
+def parseResponseHeader(response):
+    """
+    Parse response header and return HTTP code with reason
+    Example taken from WMCore pycurl_manager
+    """
+    startRegex = r"HTTP\/\d.\d\s\d{3}[^\n]*"
+    continueRegex = r"HTTP\/\d.\d\s100[^\n]*"  # Continue: client should continue its request
+    replaceRegex = r"HTTP\/\d.\d"
+
+    reason = ''
+    code = 9999
+    for row in response.split('\r'):
+        row = row.replace('\n', '')
+        if not row:
+            continue
+        response = re.search(startRegex, row)
+        if response:
+            if re.search(continueRegex, row):
+                continue
+            res = re.sub(replaceRegex, "", response.group(0)).strip()
+            code, reason = res.split(' ', 1)
+            code = int(code)
+
+    return code, reason
+
+
 class HTTPRequests(dict):
     """
     This code forks a subprocess which executes curl to communicate
@@ -162,12 +188,8 @@ def makeRequest(self, uri=None, data=None, verb='GET'):
         nRetries = max(2, self['retry'])
         for i in range(nRetries + 1):
             stdout, stderr, curlExitCode = execute_command(command=command, logger=self.logger)
+            http_code, http_reason = parseResponseHeader(stderr)
 
-            http_code, http_reason = 99999, ''
-            http_response = re.findall(r'(?<=\<\sHTTP/1.1\s)[^\n]*',stderr)
-            if http_response:
-                http_code, http_reason = http_response[-1].split(" ", 1)
-                http_code = int(http_code)
             if curlExitCode != 0 or http_code != 200:
                 if (i < 2) or (retriableError(http_code, curlExitCode) and (i < self['retry'])):
                     sleeptime = 20 * (i + 1) + random.randint(-10, 10)