From c3a53c635ab7d4e948e0b701274575b09f90b5ac Mon Sep 17 00:00:00 2001 From: ChanochShayner Date: Tue, 19 Sep 2023 09:18:28 +0300 Subject: [PATCH 1/5] adding retry for urllib3.exceptions protocolError --- .../features/fixes_integration.py | 17 ++++++++++++++++- .../common/bridgecrew/platform_integration.py | 5 +++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py index 1a07749f83f..49d0927e5ce 100644 --- a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py @@ -6,6 +6,9 @@ from itertools import groupby from typing import TYPE_CHECKING, Any +from urllib3 import PoolManager +from urllib3.exceptions import ProtocolError + from checkov.common.bridgecrew.integration_features.base_integration_feature import BaseIntegrationFeature from checkov.common.bridgecrew.integration_features.features.policy_metadata_integration import integration as metadata_integration from checkov.common.bridgecrew.platform_integration import bc_integration @@ -111,7 +114,19 @@ def _get_fixes_for_file( if not self.bc_integration.http: raise AttributeError("HTTP manager was not correctly created") - request = self.bc_integration.http.request("POST", self.fixes_url, headers=headers, body=json.dumps(payload)) # type:ignore[no-untyped-call] + try: + request = self.bc_integration.http.request("POST", self.fixes_url, headers=headers, body=json.dumps(payload)) # type:ignore[no-untyped-call] + except ProtocolError as e: + logging.error(f'Get fixes request for file {filename} failed with response code error: {e}') + if isinstance(self.bc_integration.http, PoolManager): + self.bc_integration.http.clear() + self.bc_integration.setup_http_manager( + self.bc_integration.ca_certificate, + self.bc_integration.no_cert_verify + ) + request = self.bc_integration.http.request("POST", self.fixes_url, headers=headers, body=json.dumps(payload)) # type:ignore[no-untyped-call] + else: + return None if request.status != 200: error_message = extract_error_message(request) diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 2f85a93680b..385881a9af5 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -121,6 +121,8 @@ def __init__(self) -> None: self.support_flag_enabled = False self.enable_persist_graphs = convert_str_to_bool(os.getenv('BC_ENABLE_PERSIST_GRAPHS', 'True')) self.persist_graphs_timeout = int(os.getenv('BC_PERSIST_GRAPHS_TIMEOUT', 60)) + self.ca_certificate: str | None = None + self.no_cert_verify: bool = False def set_bc_api_url(self, new_url: str) -> None: self.bc_api_url = normalize_bc_url(new_url) @@ -215,6 +217,8 @@ def setup_http_manager(self, ca_certificate: str | None = None, no_cert_verify: os.environ['REQUESTS_CA_BUNDLE'] = ca_certificate cert_reqs = 'CERT_NONE' if no_cert_verify else 'REQUIRED' logging.debug(f'Using CA cert {ca_certificate} and cert_reqs {cert_reqs}') + self.ca_certificate = ca_certificate + self.no_cert_verify = cert_reqs try: parsed_url = urllib3.util.parse_url(os.environ['https_proxy']) self.http = urllib3.ProxyManager(os.environ['https_proxy'], @@ -226,6 +230,7 @@ def setup_http_manager(self, ca_certificate: str | None = None, no_cert_verify: else: cert_reqs = 'CERT_NONE' if no_cert_verify else None logging.debug(f'Using cert_reqs {cert_reqs}') + self.no_cert_verify = cert_reqs try: parsed_url = urllib3.util.parse_url(os.environ['https_proxy']) self.http = urllib3.ProxyManager(os.environ['https_proxy'], From 30c31c25ee1f0aaf59242eba65863e8e663b00fd Mon Sep 17 00:00:00 2001 From: ChanochShayner Date: Tue, 19 Sep 2023 09:46:22 +0300 Subject: [PATCH 2/5] Mypy --- .../integration_features/features/fixes_integration.py | 2 +- checkov/common/bridgecrew/platform_integration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py index e6c87a96159..00f71e9df58 100644 --- a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py @@ -121,7 +121,7 @@ def _get_fixes_for_file( except ProtocolError as e: logging.error(f'Get fixes request for file {filename} failed with response code error: {e}') if isinstance(self.bc_integration.http, PoolManager): - self.bc_integration.http.clear() + self.bc_integration.http.clear() # type:ignore[no-untyped-call] self.bc_integration.setup_http_manager( self.bc_integration.ca_certificate, self.bc_integration.no_cert_verify diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 385881a9af5..3776d3ecc5f 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -122,7 +122,7 @@ def __init__(self) -> None: self.enable_persist_graphs = convert_str_to_bool(os.getenv('BC_ENABLE_PERSIST_GRAPHS', 'True')) self.persist_graphs_timeout = int(os.getenv('BC_PERSIST_GRAPHS_TIMEOUT', 60)) self.ca_certificate: str | None = None - self.no_cert_verify: bool = False + self.no_cert_verify: str | None = None def set_bc_api_url(self, new_url: str) -> None: self.bc_api_url = normalize_bc_url(new_url) From 29d056aed72900a76a37a2a6b4d2a7fcb56e71d3 Mon Sep 17 00:00:00 2001 From: ChanochShayner Date: Tue, 19 Sep 2023 09:52:39 +0300 Subject: [PATCH 3/5] Mypy --- .../integration_features/features/fixes_integration.py | 2 +- checkov/common/bridgecrew/platform_integration.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py index 00f71e9df58..668138abbc2 100644 --- a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py @@ -121,7 +121,7 @@ def _get_fixes_for_file( except ProtocolError as e: logging.error(f'Get fixes request for file {filename} failed with response code error: {e}') if isinstance(self.bc_integration.http, PoolManager): - self.bc_integration.http.clear() # type:ignore[no-untyped-call] + self.bc_integration.http = None self.bc_integration.setup_http_manager( self.bc_integration.ca_certificate, self.bc_integration.no_cert_verify diff --git a/checkov/common/bridgecrew/platform_integration.py b/checkov/common/bridgecrew/platform_integration.py index 3776d3ecc5f..2ab8e264c97 100644 --- a/checkov/common/bridgecrew/platform_integration.py +++ b/checkov/common/bridgecrew/platform_integration.py @@ -122,7 +122,7 @@ def __init__(self) -> None: self.enable_persist_graphs = convert_str_to_bool(os.getenv('BC_ENABLE_PERSIST_GRAPHS', 'True')) self.persist_graphs_timeout = int(os.getenv('BC_PERSIST_GRAPHS_TIMEOUT', 60)) self.ca_certificate: str | None = None - self.no_cert_verify: str | None = None + self.no_cert_verify: bool = False def set_bc_api_url(self, new_url: str) -> None: self.bc_api_url = normalize_bc_url(new_url) @@ -208,6 +208,9 @@ def setup_http_manager(self, ca_certificate: str | None = None, no_cert_verify: :param ca_certificate: an optional CA bundle to be used by both libraries. :param no_cert_verify: whether to skip SSL cert verification """ + self.ca_certificate = ca_certificate + self.no_cert_verify = no_cert_verify + ca_certificate = ca_certificate or os.getenv('BC_CA_BUNDLE') cert_reqs: str | None @@ -217,8 +220,6 @@ def setup_http_manager(self, ca_certificate: str | None = None, no_cert_verify: os.environ['REQUESTS_CA_BUNDLE'] = ca_certificate cert_reqs = 'CERT_NONE' if no_cert_verify else 'REQUIRED' logging.debug(f'Using CA cert {ca_certificate} and cert_reqs {cert_reqs}') - self.ca_certificate = ca_certificate - self.no_cert_verify = cert_reqs try: parsed_url = urllib3.util.parse_url(os.environ['https_proxy']) self.http = urllib3.ProxyManager(os.environ['https_proxy'], @@ -230,7 +231,6 @@ def setup_http_manager(self, ca_certificate: str | None = None, no_cert_verify: else: cert_reqs = 'CERT_NONE' if no_cert_verify else None logging.debug(f'Using cert_reqs {cert_reqs}') - self.no_cert_verify = cert_reqs try: parsed_url = urllib3.util.parse_url(os.environ['https_proxy']) self.http = urllib3.ProxyManager(os.environ['https_proxy'], From 17f3f94f9f0e18b28939a04c93aecfea28541cc2 Mon Sep 17 00:00:00 2001 From: ChanochShayner Date: Tue, 19 Sep 2023 09:58:03 +0300 Subject: [PATCH 4/5] Mypy --- .../integration_features/features/fixes_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py index 668138abbc2..e5e0a8c5cf8 100644 --- a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py @@ -126,7 +126,7 @@ def _get_fixes_for_file( self.bc_integration.ca_certificate, self.bc_integration.no_cert_verify ) - request = self.bc_integration.http.request("POST", self.fixes_url, headers=headers, body=json.dumps(payload)) # type:ignore[no-untyped-call] + request = self.bc_integration.http.request("POST", self.fixes_url, headers=headers, body=json.dumps(payload)) # type:ignore else: return None From 564499784955bffad5c419f03c620e87b6303869 Mon Sep 17 00:00:00 2001 From: ChanochShayner Date: Tue, 19 Sep 2023 10:55:01 +0300 Subject: [PATCH 5/5] CR fix --- .../integration_features/features/fixes_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py index e5e0a8c5cf8..4d033a7af40 100644 --- a/checkov/common/bridgecrew/integration_features/features/fixes_integration.py +++ b/checkov/common/bridgecrew/integration_features/features/fixes_integration.py @@ -118,6 +118,8 @@ def _get_fixes_for_file( try: logging.debug(f'Calling fixes API with payload: {json.dumps(payload)}, headers: {headers}, url: {self.fixes_url}') request = self.bc_integration.http.request("POST", self.fixes_url, headers=headers, body=json.dumps(payload)) # type:ignore[no-untyped-call] + + # When running via IDE we can fail here in case of running with -d when the poolManager is broken except ProtocolError as e: logging.error(f'Get fixes request for file {filename} failed with response code error: {e}') if isinstance(self.bc_integration.http, PoolManager):