From 9c8153c028c5ebb8b2646a0cbe995facdc402431 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 14:17:29 -0400 Subject: [PATCH 01/29] Linting: anomalous-backslash-in-string: Add r prefix to regex string --- src/bos/common/utils.py | 2 +- src/bos/reporter/status_reporter/__main__.py | 2 +- src/bos/server/controllers/v2/sessions.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bos/common/utils.py b/src/bos/common/utils.py index 98256b86..e7bb85f0 100644 --- a/src/bos/common/utils.py +++ b/src/bos/common/utils.py @@ -30,7 +30,7 @@ from requests.packages.urllib3.util.retry import Retry PROTOCOL = 'http' -TIME_DURATION_PATTERN = re.compile("^(\d+?)(\D+?)$", re.M|re.S) +TIME_DURATION_PATTERN = re.compile(r"^(\d+?)(\D+?)$", re.M|re.S) # Common date and timestamps functions so that timezones and formats are handled consistently. def get_current_time() -> datetime.datetime: diff --git a/src/bos/reporter/status_reporter/__main__.py b/src/bos/reporter/status_reporter/__main__.py index efd6a512..c87ab416 100644 --- a/src/bos/reporter/status_reporter/__main__.py +++ b/src/bos/reporter/status_reporter/__main__.py @@ -50,7 +50,7 @@ _stream_handler.setLevel(LOG_LEVEL) PROJECT_LOGGER.addHandler(_stream_handler) PROJECT_LOGGER.setLevel(LOG_LEVEL) -TIME_DURATION_PATTERN = re.compile("^(\d+?)(\D+?)$", re.M | re.S) +TIME_DURATION_PATTERN = re.compile(r"^(\d+?)(\D+?)$", re.M | re.S) # The percentage of the total Time To Live (TTL) to wait before reporting status, e.g. # a state TTL of 4 hours with a ratio of .75 means nodes report every 3 hours. diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index 5ce0edc9..f1a7598b 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -372,7 +372,7 @@ def _get_v2_session_status(session_key, session=None): def _age_to_timestamp(age): delta = {} for interval in ['weeks', 'days', 'hours', 'minutes']: - result = re.search('(\d+)\w*{}'.format(interval[0]), age, re.IGNORECASE) + result = re.search(r'(\d+)\w*{}'.format(interval[0]), age, re.IGNORECASE) if result: delta[interval] = int(result.groups()[0]) delta = timedelta(**delta) From 8bdad0852e9443cb8dc09497a4f797bbc0505644 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 13:32:03 -0400 Subject: [PATCH 02/29] Linting: bad-indentation: Correct indentation --- src/bos/server/controllers/v2/boot_set.py | 2 +- src/bos/server/controllers/v2/components.py | 30 ++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/bos/server/controllers/v2/boot_set.py b/src/bos/server/controllers/v2/boot_set.py index bbcb865e..b12feeee 100644 --- a/src/bos/server/controllers/v2/boot_set.py +++ b/src/bos/server/controllers/v2/boot_set.py @@ -119,7 +119,7 @@ def validate_boot_sets(session_template: dict, warning_flag = True warn_msg = warn_msg + msg if warning_flag: - return BOOT_SET_WARNING, warn_msg + return BOOT_SET_WARNING, warn_msg return BOOT_SET_SUCCESS, "Valid" diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index e81805ce..8bfc5d9f 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -394,21 +394,21 @@ def patch_v2_component(component_id): def validate_actual_state_change_is_allowed(component_id): - current_data = DB.get(component_id) - if not current_data["enabled"]: - # This component is not being managed on by BOS - return True - if _calculate_status(current_data) == Status.stable: - # BOS believes the component is in the correct state - return True - if current_data["last_action"]["action"] == Action.power_on: - # BOS just powered-on the component and is waiting for the new state to be reported - return True - # The component is being actively changed by BOS, and is going to be powered off or - # is in a state where the next action hasn't been determined. Allowing the actual - # state to be updated can interfere with BOS' ability to determine the next action. - # e.g. When the actual_state is deleted by the setup operator to trigger a reboot - return False + current_data = DB.get(component_id) + if not current_data["enabled"]: + # This component is not being managed on by BOS + return True + if _calculate_status(current_data) == Status.stable: + # BOS believes the component is in the correct state + return True + if current_data["last_action"]["action"] == Action.power_on: + # BOS just powered-on the component and is waiting for the new state to be reported + return True + # The component is being actively changed by BOS, and is going to be powered off or + # is in a state where the next action hasn't been determined. Allowing the actual + # state to be updated can interfere with BOS' ability to determine the next action. + # e.g. When the actual_state is deleted by the setup operator to trigger a reboot + return False @tenant_error_handler From 6aef65d771d1baab5ac895d924c51a49d9f8f2b9 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 15:12:01 -0400 Subject: [PATCH 03/29] Linting: invalid-envvar-default: Replace invalid env var default value type --- src/bos/operators/utils/liveness/timestamp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bos/operators/utils/liveness/timestamp.py b/src/bos/operators/utils/liveness/timestamp.py index 37174f7a..f2b048c2 100644 --- a/src/bos/operators/utils/liveness/timestamp.py +++ b/src/bos/operators/utils/liveness/timestamp.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -39,5 +39,5 @@ def max_age(self): This value is returned as a timedelta object. """ - computation_time = timedelta(seconds=int(os.getenv('LIVENESS_DELTA_MAX', 20))) + computation_time = timedelta(seconds=int(os.getenv('LIVENESS_DELTA_MAX', "20"))) return computation_time From 657cc27bc7709a8ca308705aa56cbe0817e1fbd1 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 14:32:59 -0400 Subject: [PATCH 04/29] Linting: pointless-string-statement: Cleanup misplaced string statements --- src/bos/server/controllers/v2/options.py | 2 +- .../server/controllers/v2/sessiontemplates.py | 26 +++++++++---------- src/bos/server/redis_db_utils.py | 4 +-- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index cdf78303..61d41fd8 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -56,7 +56,7 @@ def _init(): log_level_updater = threading.Thread(target=check_v2_logging_level, args=()) log_level_updater.start() - """ Cleanup old options """ + # Cleanup old options while True: try: data = DB.get(OPTIONS_KEY) diff --git a/src/bos/server/controllers/v2/sessiontemplates.py b/src/bos/server/controllers/v2/sessiontemplates.py index 6f86d3ea..25955ea4 100644 --- a/src/bos/server/controllers/v2/sessiontemplates.py +++ b/src/bos/server/controllers/v2/sessiontemplates.py @@ -100,13 +100,12 @@ def put_v2_sessiontemplate(session_template_id): # noqa: E501 template_data = data try: - """Convert the JSON request data into a SessionTemplate object. - Any exceptions caught here would be generated from the model - (i.e. bos.server.models.session_template). - An example is an exception for a session template name that - does not conform to Kubernetes naming convention. - In this case return 400 with a description of the specific error. - """ + # Convert the JSON request data into a SessionTemplate object. + # Any exceptions caught here would be generated from the model + # (i.e. bos.server.models.session_template). + # An example is an exception for a session template name that + # does not conform to Kubernetes naming convention. + # In this case return 400 with a description of the specific error. SessionTemplate.from_dict(template_data) except Exception as err: LOGGER.error("Error creating session template: %s", exc_type_msg(err)) @@ -214,13 +213,12 @@ def patch_v2_sessiontemplate(session_template_id): template_data = data try: - """Convert the JSON request data into a SessionTemplate object. - Any exceptions caught here would be generated from the model - (i.e. bos.server.models.session_template). - An example is an exception for a session template name that - does not confirm to Kubernetes naming convention. - In this case return 400 with a description of the specific error. - """ + # Convert the JSON request data into a SessionTemplate object. + # Any exceptions caught here would be generated from the model + # (i.e. bos.server.models.session_template). + # An example is an exception for a session template name that + # does not confirm to Kubernetes naming convention. + # In this case return 400 with a description of the specific error. SessionTemplate.from_dict(template_data) except Exception as err: LOGGER.error("Error patching session template: %s", exc_type_msg(err)) diff --git a/src/bos/server/redis_db_utils.py b/src/bos/server/redis_db_utils.py index faf906bc..4260dcad 100644 --- a/src/bos/server/redis_db_utils.py +++ b/src/bos/server/redis_db_utils.py @@ -104,8 +104,8 @@ def put(self, key, new_data): return self.get(key) def patch(self, key, new_data, data_handler=None): - """Patch data in the database.""" - """data_handler provides a way to operate on the full patched data""" + """Patch data in the database. + data_handler provides a way to operate on the full patched data""" datastr = self.client.get(key) data = json.loads(datastr) data = self._update(data, new_data) From 57f5d69ae7c0c08bf8b1eca1cd27f85f4b15743f Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 14:21:50 -0400 Subject: [PATCH 05/29] Linting: raise-missing-from: Raise exceptions from 'provoking' exceptions --- .../utils/boot_image_metadata/s3_boot_image_metadata.py | 2 +- src/bos/reporter/components/state.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py index ff3f52fd..497ae406 100644 --- a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py +++ b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py @@ -81,7 +81,7 @@ def metadata(self): return self.boot_artifacts.manifest_json except (ClientError, S3MissingConfiguration) as error: LOGGER.error("Unable to read %s -- Error: %s", self._boot_set.get('path', ''), exc_type_msg(error)) - raise BootImageMetaDataBadRead(error) + raise BootImageMetaDataBadRead(error) from error @property def kernel(self): diff --git a/src/bos/reporter/components/state.py b/src/bos/reporter/components/state.py index 20251cb0..eabc5c3a 100644 --- a/src/bos/reporter/components/state.py +++ b/src/bos/reporter/components/state.py @@ -70,7 +70,7 @@ def patch_component(component, properties, session=None): if response.status_code == 404: try: json_response = json.loads(response.text) - raise UnknownComponent(json_response['detail']) + raise UnknownComponent(json_response['detail']) from hpe except json.JSONDecodeError as jde: raise UnrecognizedResponse("BOS returned a non-json response: %s\n%s" % (response.text, jde)) from jde LOGGER.warning("Unexpected response from '%s':\n%s: %s", component_endpoint, response.status_code, response.text) From 78c82199d3963dbcf226293a64dac2441dca49dc Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:15:01 -0400 Subject: [PATCH 06/29] Linting: no-else-return/no-else-raise: Remove/convert unnecessary elif/else clauses --- src/bos/common/tenant_utils.py | 3 +- src/bos/operators/session_setup.py | 18 +++----- .../utils/boot_image_metadata/factory.py | 3 +- src/bos/operators/utils/clients/bos/base.py | 10 +---- .../operators/utils/clients/bos/options.py | 5 +-- src/bos/operators/utils/clients/s3.py | 3 +- src/bos/operators/utils/rootfs/__init__.py | 5 +-- src/bos/operators/utils/rootfs/baserootfs.py | 7 +-- src/bos/server/controllers/v2/components.py | 43 ++++++++----------- src/bos/server/controllers/v2/sessions.py | 3 +- src/bos/server/redis_db_utils.py | 3 +- 11 files changed, 38 insertions(+), 65 deletions(-) diff --git a/src/bos/common/tenant_utils.py b/src/bos/common/tenant_utils.py index 0ed4b19f..0fb9c47c 100644 --- a/src/bos/common/tenant_utils.py +++ b/src/bos/common/tenant_utils.py @@ -81,8 +81,7 @@ def get_tenant_data(tenant, session=None): LOGGER.error("Failed getting tenant data from tapms: %s", exc_type_msg(e)) if response.status_code == 404: raise InvalidTenantException(f"Data not found for tenant {tenant}") from e - else: - raise + raise return response.json() diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 06fea441..9dde0472 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -137,10 +137,10 @@ def _setup_components(self): raise SessionSetupException("No nodes were found to act upon.") except Exception as err: raise SessionSetupException(err) from err - else: - self._log(LOGGER.info, 'Found %d components that require updates', len(data)) - self._log(LOGGER.debug, f'Updated components: {data}') - self.bos_client.components.update_components(data) + # No exception raised by previous block + self._log(LOGGER.info, 'Found %d components that require updates', len(data)) + self._log(LOGGER.debug, f'Updated components: {data}') + self.bos_client.components.update_components(data) return list(set(all_component_ids)) def _get_boot_set_component_list(self, boot_set) -> Set[str]: @@ -319,13 +319,9 @@ def _operate(self, component_id, state): def _generate_desired_state(self, boot_set, staged=False): if self.operation_type == "shutdown": - if staged: - return EMPTY_STAGED_STATE - else: - return EMPTY_DESIRED_STATE - else: - state = self._get_state_from_boot_set(boot_set) - return state + return EMPTY_STAGED_STATE if staged else EMPTY_DESIRED_STATE + state = self._get_state_from_boot_set(boot_set) + return state def _get_state_from_boot_set(self, boot_set): """ diff --git a/src/bos/operators/utils/boot_image_metadata/factory.py b/src/bos/operators/utils/boot_image_metadata/factory.py index c9aef39e..cbb3032d 100644 --- a/src/bos/operators/utils/boot_image_metadata/factory.py +++ b/src/bos/operators/utils/boot_image_metadata/factory.py @@ -46,5 +46,4 @@ def __call__(self): if path_type: if path_type == 's3': return S3BootImageMetaData(self.boot_set) - else: - raise BootImageMetaDataUnknown(f"No BootImageMetaData class for type {path_type}") + raise BootImageMetaDataUnknown(f"No BootImageMetaData class for type {path_type}") diff --git a/src/bos/operators/utils/clients/bos/base.py b/src/bos/operators/utils/clients/bos/base.py index 23b41e03..a7367622 100644 --- a/src/bos/operators/utils/clients/bos/base.py +++ b/src/bos/operators/utils/clients/bos/base.py @@ -119,10 +119,7 @@ def delete_items(self, **kwargs): LOGGER.debug("DELETE %s with params=%s", self.base_url, kwargs) response = self.session.delete(self.base_url, params=kwargs) response.raise_for_status() - if response.text: - return json.loads(response.text) - else: - return None + return json.loads(response.text) if response.text else None class BaseBosTenantAwareEndpoint(BaseBosEndpoint): @@ -196,7 +193,4 @@ def delete_items(self, **kwargs): LOGGER.debug("DELETE %s with params=%s", self.base_url, kwargs) response = self.session.delete(self.base_url, params=kwargs, headers=headers) response.raise_for_status() - if response.text: - return json.loads(response.text) - else: - return None + return json.loads(response.text) if response.text else None diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/operators/utils/clients/bos/options.py index df06a4da..826f4f68 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/operators/utils/clients/bos/options.py @@ -66,10 +66,9 @@ def _get_options(self): def get_option(self, key, value_type, default): if key in self.options: return value_type(self.options[key]) - elif default: + if default: return value_type(default) - else: - raise KeyError('Option {} not found and no default exists'.format(key)) + raise KeyError('Option {} not found and no default exists'.format(key)) @property def logging_level(self): diff --git a/src/bos/operators/utils/clients/s3.py b/src/bos/operators/utils/clients/s3.py index 2d16ab50..e6dd0f41 100644 --- a/src/bos/operators/utils/clients/s3.py +++ b/src/bos/operators/utils/clients/s3.py @@ -86,8 +86,7 @@ def bucket(self): def key(self): if self._parsed.query: return self._parsed.path.lstrip('/') + '?' + self._parsed.query - else: - return self._parsed.path.lstrip('/') + return self._parsed.path.lstrip('/') @property def url(self): diff --git a/src/bos/operators/utils/rootfs/__init__.py b/src/bos/operators/utils/rootfs/__init__.py index d00f90c5..d101edec 100644 --- a/src/bos/operators/utils/rootfs/__init__.py +++ b/src/bos/operators/utils/rootfs/__init__.py @@ -76,10 +76,7 @@ def __str__(self): fields.append(rootfs_provider_passthrough) stripped_fields = [field for field in fields if field] - if stripped_fields: - return "root={}".format(self.DELIMITER.join(fields)) - else: - return '' + return "root={}".format(self.DELIMITER.join(fields)) if stripped_fields else '' @property def provider_field(self): diff --git a/src/bos/operators/utils/rootfs/baserootfs.py b/src/bos/operators/utils/rootfs/baserootfs.py index 31189143..89152c48 100644 --- a/src/bos/operators/utils/rootfs/baserootfs.py +++ b/src/bos/operators/utils/rootfs/baserootfs.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2022-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2022-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -51,7 +51,4 @@ def nmd_field(self): fields.append("url=%s" % self.provider_field) if self.provider_field_id: fields.append("etag=%s" % self.provider_field_id) - if fields: - return "nmd_data={}".format(",".join(fields)) - else: - return '' \ No newline at end of file + return "nmd_data={}".format(",".join(fields)) if fields else '' diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 8bfc5d9f..ae8823e3 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -124,25 +124,22 @@ def _calculate_status(data): if last_action == Action.power_on and not data.get('last_action', {}).get('failed', False): LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_on_called}") return Status.power_on_called - else: - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_on_pending}") - return Status.power_on_pending - elif phase == Phase.powering_off: + LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_on_pending}") + return Status.power_on_pending + if phase == Phase.powering_off: if last_action == Action.power_off_gracefully: LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_gracefully_called}") return Status.power_off_gracefully_called - elif last_action == Action.power_off_forcefully: + if last_action == Action.power_off_forcefully: LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_forcefully_called}") return Status.power_off_forcefully_called - else: - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_pending}") - return Status.power_off_pending - elif phase == Phase.configuring: + LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_pending}") + return Status.power_off_pending + if phase == Phase.configuring: LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.configuring}") return Status.configuring - else: - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.stable}") - return Status.stable + LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.stable}") + return Status.stable def _matches_filter(data, enabled, session, staged_session, phase, status): @@ -224,7 +221,7 @@ def patch_v2_components(): LOGGER.error("%s: %s", msg, exc_type_msg(err)) return connexion.problem(status=400, title=msg,detail=str(err)) return patch_v2_components_list(data) - elif type(data) == dict: + if type(data) == dict: try: # This call is just to ensure that the data # coming in is valid per the API schema @@ -276,7 +273,7 @@ def patch_v2_components_dict(data): return connexion.problem( status=400, title="Only one filter may be provided.", detail="Only one filter may be provided.") - elif ids: + if ids: try: id_list = ids.split(',') except Exception as err: @@ -457,14 +454,13 @@ def post_v2_apply_staged(): def _apply_tenant_limit(component_list): tenant = get_tenant_from_header() - if tenant: - tenant_components = get_tenant_component_set(tenant) - component_set = set(component_list) - allowed_components = component_set.intersection(tenant_components) - rejected_components = component_set.difference(tenant_components) - return list(allowed_components), list(rejected_components) - else: + if not tenant: return component_list, [] + tenant_components = get_tenant_component_set(tenant) + component_set = set(component_list) + allowed_components = component_set.intersection(tenant_components) + rejected_components = component_set.difference(tenant_components) + return list(allowed_components), list(rejected_components) def _is_valid_tenant_component(component_id): @@ -472,9 +468,8 @@ def _is_valid_tenant_component(component_id): if tenant: tenant_components = get_tenant_component_set(tenant) return component_id in tenant_components - else: - # For an empty tenant, all components are valid - return True + # For an empty tenant, all components are valid + return True def _apply_staged(component_id, clear_staged=False): diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index f1a7598b..7af9ef6e 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -76,8 +76,7 @@ def post_v2_session(): # noqa: E501 msg = "Session Template Name invalid: {}".format(template_name) LOGGER.error(msg) return msg, 400 - else: - session_template, _ = session_template_response + session_template, _ = session_template_response # Validate health/validity of the sessiontemplate before creating a session error_code, msg = validate_boot_sets(session_template, session_create.operation, template_name) diff --git a/src/bos/server/redis_db_utils.py b/src/bos/server/redis_db_utils.py index 4260dcad..21f4fdb5 100644 --- a/src/bos/server/redis_db_utils.py +++ b/src/bos/server/redis_db_utils.py @@ -57,8 +57,7 @@ def _get_db_id(self, db): """Converts a db name to the id used by Redis.""" if isinstance(db, int): return db - else: - return DATABASES.index(db) + return DATABASES.index(db) def _get_client(self, db_id): """Create a connection with the database.""" From e694e5b9430c59388a53f87d770bbc999beb868c Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:21:14 -0400 Subject: [PATCH 07/29] Linting: consider-using-/unidiomatic-typecheck/use-a-generator/use-maxsplit-arg consider-using-in: Merge comparisons by using in statement consider-using-dict-items: Iterate over dict keys and values using items method consider-using-with: Use with when opening file consider-using-set-comprehension: Use set comprehension instead of list comprehension + call to set function unidiomatic-typecheck: Use isinstance to check object type use-a-generator: Use a generator instead of creating a list object use-maxsplit-arg: Use maxsplit when accessing only last element of split --- src/bos/operators/base.py | 2 +- src/bos/operators/filters/filters.py | 4 ++-- src/bos/operators/session_setup.py | 2 +- .../operators/utils/clients/bos/components.py | 4 ++-- src/bos/operators/utils/clients/bos/options.py | 2 +- src/bos/operators/utils/clients/bos/sessions.py | 4 ++-- src/bos/operators/utils/clients/hsm.py | 4 ++-- src/bos/reporter/components/__init__.py | 4 ++-- src/bos/server/controllers/v2/base.py | 16 ++++------------ src/bos/server/controllers/v2/components.py | 6 +++--- src/bos/server/controllers/v2/options.py | 4 ++-- 11 files changed, 22 insertions(+), 30 deletions(-) diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 7448a658..7a08a12d 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -291,7 +291,7 @@ def _init_logging() -> None: requested_log_level = os.environ.get('BOS_OPERATOR_LOG_LEVEL', 'INFO') log_level = logging.getLevelName(requested_log_level) - if type(log_level) != int: + if not isinstance(log_level, int): LOGGER.warning('Log level %r is not valid. Falling back to INFO', requested_log_level) log_level = logging.INFO logging.basicConfig(level=log_level, format=log_format) diff --git a/src/bos/operators/filters/filters.py b/src/bos/operators/filters/filters.py index b6ae5d26..e4cf1f0c 100644 --- a/src/bos/operators/filters/filters.py +++ b/src/bos/operators/filters/filters.py @@ -232,7 +232,7 @@ class DesiredBootStateIsNone(LocalFilter): def _match(self, component: dict) -> bool: desired_state = component.get('desired_state', {}) desired_boot_state = desired_state.get('boot_artifacts', {}) - if not desired_boot_state or not any([bool(v) for v in desired_boot_state.values()]): + if not desired_boot_state or not any(bool(v) for v in desired_boot_state.values()): return True return False @@ -285,7 +285,7 @@ def _match(self, component: dict) -> bool: # The timestamp field doesn't count as a set record we particularly care about if 'timestamp' in actual_state_boot_artifacts: del actual_state_boot_artifacts['timestamp'] - if any([bool(v) for v in actual_state_boot_artifacts.values()]): + if any(bool(v) for v in actual_state_boot_artifacts.values()): return True return False diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 9dde0472..003be82c 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -253,7 +253,7 @@ def _apply_limit(self, nodes): op = limit_node_set.union limit_nodes = set([limit]) - if limit == 'all' or limit == '*': + if limit in {'all', '*'}: limit_nodes = nodes elif limit in self.inventory: limit_nodes = self.inventory[limit] diff --git a/src/bos/operators/utils/clients/bos/components.py b/src/bos/operators/utils/clients/bos/components.py index 37a72962..2e17f7d4 100644 --- a/src/bos/operators/utils/clients/bos/components.py +++ b/src/bos/operators/utils/clients/bos/components.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -29,7 +29,7 @@ class ComponentEndpoint(BaseBosEndpoint): - ENDPOINT = __name__.lower().split('.')[-1] + ENDPOINT = __name__.lower().rsplit('.', maxsplit=1)[-1] def get_component(self, component_id): return self.get_item(component_id) diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/operators/utils/clients/bos/options.py index 826f4f68..e2b10d9a 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/operators/utils/clients/bos/options.py @@ -30,7 +30,7 @@ from bos.operators.utils.clients.bos.base import BASE_ENDPOINT LOGGER = logging.getLogger('bos.operators.utils.clients.bos.options') -ENDPOINT = "%s/%s" % (BASE_ENDPOINT, __name__.lower().split('.')[-1]) +ENDPOINT = "%s/%s" % (BASE_ENDPOINT, __name__.lower().rsplit('.', maxsplit=1)[-1]) class Options: diff --git a/src/bos/operators/utils/clients/bos/sessions.py b/src/bos/operators/utils/clients/bos/sessions.py index 8ac83e8c..3d958e1f 100644 --- a/src/bos/operators/utils/clients/bos/sessions.py +++ b/src/bos/operators/utils/clients/bos/sessions.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -29,7 +29,7 @@ class SessionEndpoint(BaseBosTenantAwareEndpoint): - ENDPOINT = __name__.lower().split('.')[-1] + ENDPOINT = __name__.lower().rsplit('.', maxsplit=1)[-1] def get_session(self, session_id, tenant): return self.get_item(session_id, tenant) diff --git a/src/bos/operators/utils/clients/hsm.py b/src/bos/operators/utils/clients/hsm.py index f72fb3b6..77e5b54a 100644 --- a/src/bos/operators/utils/clients/hsm.py +++ b/src/bos/operators/utils/clients/hsm.py @@ -74,8 +74,8 @@ def read_all_node_xnames(): LOGGER.error("Non-JSON response from HSM: %s", response.text) raise HWStateManagerException(jde) from jde try: - return set([component['ID'] for component in json_body['Components'] - if component.get('Type', None) == 'Node']) + return {[component['ID'] for component in json_body['Components'] + if component.get('Type', None) == 'Node']} except KeyError as ke: LOGGER.error("Unexpected API response from HSM: %s", exc_type_msg(ke)) raise HWStateManagerException(ke) from ke diff --git a/src/bos/reporter/components/__init__.py b/src/bos/reporter/components/__init__.py index 95879237..cbc473b2 100644 --- a/src/bos/reporter/components/__init__.py +++ b/src/bos/reporter/components/__init__.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -24,7 +24,7 @@ from bos.reporter import BOSException from bos.reporter import ENDPOINT as BOS_ENDPOINT -ENDPOINT = "%s/%s" % (BOS_ENDPOINT, __name__.split('.')[-1]) +ENDPOINT = "%s/%s" % (BOS_ENDPOINT, __name__.rsplit('.', maxsplit=1)[-1]) class BOSComponentException(BOSException): diff --git a/src/bos/server/controllers/v2/base.py b/src/bos/server/controllers/v2/base.py index 620e9a71..7b0d54fc 100644 --- a/src/bos/server/controllers/v2/base.py +++ b/src/bos/server/controllers/v2/base.py @@ -52,22 +52,14 @@ def calc_version(details): # parse open API spec file from docker image or local repository openapispec_f = '/app/lib/bos/server/openapi/openapi.yaml' - f = None try: - f = open(openapispec_f, 'r') + with open(openapispec_f, 'r') as f: + openapispec_map = yaml.safe_load(f) + major, minor, patch = openapispec_map['info']['version'].split('.') + return Version(major=major, minor=minor, patch=patch, links=links) except IOError as e: LOGGER.debug('error opening "%s" file: %s', openapispec_f, exc_type_msg(e)) - openapispec_map = yaml.safe_load(f) - f.close() - major, minor, patch = openapispec_map['info']['version'].split('.') - return Version( - major=major, - minor=minor, - patch=patch, - links=links, - ) - def get_v2(): LOGGER.debug("GET /v2 invoked get_v2") diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index ae8823e3..52da2008 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -211,7 +211,7 @@ def patch_v2_components(): LOGGER.debug("type=%s", type(data)) LOGGER.debug("Received: %s", data) - if type(data) == list: + if isinstance(data, list): try: # This call is just to ensure that the data # coming in is valid per the API schema @@ -221,7 +221,7 @@ def patch_v2_components(): LOGGER.error("%s: %s", msg, exc_type_msg(err)) return connexion.problem(status=400, title=msg,detail=str(err)) return patch_v2_components_list(data) - if type(data) == dict: + if isinstance(data, dict): try: # This call is just to ensure that the data # coming in is valid per the API schema @@ -590,7 +590,7 @@ def del_timestamp(data: dict): def _set_last_updated(data): timestamp = get_current_timestamp() for section in ['actual_state', 'desired_state', 'staged_state', 'last_action']: - if section in data and type(data[section]) == dict and data[section].keys() != {"bss_token"}: + if section in data and isinstance(data[section], dict) and data[section].keys() != {"bss_token"}: data[section]['last_updated'] = timestamp return data diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index 61d41fd8..459812eb 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -101,9 +101,9 @@ def _check_defaults(data): if not data: data = {} put = True - for key in DEFAULTS: + for key, default_value in DEFAULTS.items(): if key not in data: - data[key] = DEFAULTS[key] + data[key] = default_value put = True if put: return DB.put(OPTIONS_KEY, data) From 19e29bcee7cc8db79106b72e5035611e5cf4b82b Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:22:42 -0400 Subject: [PATCH 08/29] Linting: superfluous-parens/unnecessary-pass/useless-object-inheritance/useless-return: Remove superfluous code --- src/bos/operators/base.py | 1 - src/bos/operators/session_setup.py | 1 - src/bos/operators/utils/boot_image_metadata/__init__.py | 6 +++--- src/bos/operators/utils/boot_image_metadata/factory.py | 2 +- src/bos/operators/utils/clients/bos/base.py | 2 +- src/bos/operators/utils/clients/bos/sessions_status.py | 2 +- src/bos/operators/utils/clients/hsm.py | 2 +- src/bos/operators/utils/clients/pcs.py | 2 +- src/bos/operators/utils/clients/s3.py | 2 +- src/bos/operators/utils/rootfs/__init__.py | 2 +- src/bos/operators/utils/rootfs/factory.py | 4 ++-- src/bos/server/controllers/v2/components.py | 1 - src/bos/server/dbs/boot_artifacts.py | 1 - 13 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 7a08a12d..bec73b8a 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -53,7 +53,6 @@ class MissingSessionData(BaseOperatorException): Operators are expected to update the session data, if they are updating a component's desired state. """ - pass class BaseOperator(ABC): diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 003be82c..46d4891f 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -45,7 +45,6 @@ class SessionSetupException(Exception): """ The Session Set-up experienced a fatal error """ - pass class SessionSetupOperator(BaseOperator): diff --git a/src/bos/operators/utils/boot_image_metadata/__init__.py b/src/bos/operators/utils/boot_image_metadata/__init__.py index ba246ab7..c27e84a4 100644 --- a/src/bos/operators/utils/boot_image_metadata/__init__.py +++ b/src/bos/operators/utils/boot_image_metadata/__init__.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -21,7 +21,8 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -class BootImageMetaData(object): + +class BootImageMetaData: def __init__(self, boot_set): """ Base class for BootImage Metadata object @@ -70,4 +71,3 @@ class BootImageMetaDataBadRead(Exception): """ The metadata for the boot image could not be read/retrieved. """ - pass diff --git a/src/bos/operators/utils/boot_image_metadata/factory.py b/src/bos/operators/utils/boot_image_metadata/factory.py index cbb3032d..13a86982 100644 --- a/src/bos/operators/utils/boot_image_metadata/factory.py +++ b/src/bos/operators/utils/boot_image_metadata/factory.py @@ -33,7 +33,7 @@ class BootImageMetaDataUnknown(Exception): Raised when a user requests a Provider provisioning mechanism that is not known """ -class BootImageMetaDataFactory(object): +class BootImageMetaDataFactory: """ Conditionally create new instances of the BootImageMetadata based on the type of the BootImageMetaData specified diff --git a/src/bos/operators/utils/clients/bos/base.py b/src/bos/operators/utils/clients/bos/base.py index a7367622..5acb769f 100644 --- a/src/bos/operators/utils/clients/bos/base.py +++ b/src/bos/operators/utils/clients/bos/base.py @@ -55,7 +55,7 @@ def wrap(*args, **kwargs): return wrap -class BaseBosEndpoint(object): +class BaseBosEndpoint: """ This base class provides generic access to the BOS API. The individual endpoint needs to be overridden for a specific endpoint. diff --git a/src/bos/operators/utils/clients/bos/sessions_status.py b/src/bos/operators/utils/clients/bos/sessions_status.py index df09d798..1400e205 100644 --- a/src/bos/operators/utils/clients/bos/sessions_status.py +++ b/src/bos/operators/utils/clients/bos/sessions_status.py @@ -31,7 +31,7 @@ LOGGER = logging.getLogger('bos.operators.utils.clients.bos.sessions_status') -class SessionStatusEndpoint(object): +class SessionStatusEndpoint: ENDPOINT = 'sessions' def __init__(self): diff --git a/src/bos/operators/utils/clients/hsm.py b/src/bos/operators/utils/clients/hsm.py index 77e5b54a..5b5b696a 100644 --- a/src/bos/operators/utils/clients/hsm.py +++ b/src/bos/operators/utils/clients/hsm.py @@ -146,7 +146,7 @@ def get_components(node_list, enabled=None) -> dict[str,list[dict]]: return components -class Inventory(object): +class Inventory: """ Inventory handles the generation of a hardware inventory in a similar manner to how the dynamic inventory is generated for CFS. To reduce the number of calls to HSM, everything is diff --git a/src/bos/operators/utils/clients/pcs.py b/src/bos/operators/utils/clients/pcs.py index 3445f116..bebe2703 100644 --- a/src/bos/operators/utils/clients/pcs.py +++ b/src/bos/operators/utils/clients/pcs.py @@ -151,7 +151,7 @@ def status(nodes, session=None, **kwargs): status_bucket[power_status_entry['error']].add(xname) continue power_status = power_status_entry.get('powerState', '').lower() - if not(all([power_status, xname])): + if not all([power_status, xname]): continue status_bucket[power_status].add(xname) return status_bucket diff --git a/src/bos/operators/utils/clients/s3.py b/src/bos/operators/utils/clients/s3.py index e6dd0f41..2378bd32 100644 --- a/src/bos/operators/utils/clients/s3.py +++ b/src/bos/operators/utils/clients/s3.py @@ -70,7 +70,7 @@ class S3ObjectNotFound(Exception): """ -class S3Url(object): +class S3Url: """ https://stackoverflow.com/questions/42641315/s3-urls-get-bucket-name-and-path/42641363 """ diff --git a/src/bos/operators/utils/rootfs/__init__.py b/src/bos/operators/utils/rootfs/__init__.py index d101edec..5ebf5986 100644 --- a/src/bos/operators/utils/rootfs/__init__.py +++ b/src/bos/operators/utils/rootfs/__init__.py @@ -38,7 +38,7 @@ class ProviderNotImplemented(Exception): """ -class RootfsProvider(object): +class RootfsProvider: PROTOCOL = None DELIMITER = ':' """ diff --git a/src/bos/operators/utils/rootfs/factory.py b/src/bos/operators/utils/rootfs/factory.py index a3aa2a64..442bbe05 100644 --- a/src/bos/operators/utils/rootfs/factory.py +++ b/src/bos/operators/utils/rootfs/factory.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -28,7 +28,7 @@ LOGGER = logging.getLogger(__name__) -class ProviderFactory(object): +class ProviderFactory: """ Conditionally creates new instances of rootfilesystem providers based on a given agent instance. diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 52da2008..9d966529 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -584,7 +584,6 @@ def del_timestamp(data: dict): del data['actual_state']['boot_artifacts']['timestamp'] except KeyError: pass - return None def _set_last_updated(data): diff --git a/src/bos/server/dbs/boot_artifacts.py b/src/bos/server/dbs/boot_artifacts.py index ec4c5d35..ffb3f6cd 100644 --- a/src/bos/server/dbs/boot_artifacts.py +++ b/src/bos/server/dbs/boot_artifacts.py @@ -38,7 +38,6 @@ class BssTokenUnknown(BssTokenException): """ The BSS Token is not present in the database. """ - pass def record_boot_artifacts(token: str, From 333b542447b2a68d618f062f2b0aeaa29eb6be2a Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:24:18 -0400 Subject: [PATCH 09/29] Linting: deprecated-method Replace logging warn() with warning() Replace threading currentThread() with current_thread() --- src/bos/operators/base.py | 2 +- .../boot_image_metadata/s3_boot_image_metadata.py | 12 ++++++------ src/bos/server/controllers/v2/boot_set.py | 2 +- src/bos/server/controllers/v2/components.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index bec73b8a..bb121c16 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -41,7 +41,7 @@ from bos.operators.utils.liveness.timestamp import Timestamp LOGGER = logging.getLogger('bos.operators.base') -MAIN_THREAD = threading.currentThread() +MAIN_THREAD = threading.current_thread() class BaseOperatorException(Exception): diff --git a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py index 497ae406..2b144baf 100644 --- a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py +++ b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py @@ -46,27 +46,27 @@ def __init__(self, boot_set): try: self.artifact_summary['kernel'] = self.kernel_path except ArtifactNotFound as err: - LOGGER.warn(exc_type_msg(err)) + LOGGER.warning(exc_type_msg(err)) try: self.artifact_summary['initrd'] = self.initrd_path except ArtifactNotFound as err: - LOGGER.warn(exc_type_msg(err)) + LOGGER.warning(exc_type_msg(err)) try: self.artifact_summary['rootfs'] = self.rootfs_path except ArtifactNotFound as err: - LOGGER.warn(exc_type_msg(err)) + LOGGER.warning(exc_type_msg(err)) try: self.artifact_summary['rootfs_etag'] = self.rootfs_etag except ArtifactNotFound as err: - LOGGER.warn(exc_type_msg(err)) + LOGGER.warning(exc_type_msg(err)) try: self.artifact_summary['boot_parameters'] = self.boot_parameters_path except ArtifactNotFound as err: - LOGGER.warn(exc_type_msg(err)) + LOGGER.warning(exc_type_msg(err)) try: self.artifact_summary['boot_parameters_etag'] = self.boot_parameters_etag except ArtifactNotFound as err: - LOGGER.warn(exc_type_msg(err)) + LOGGER.warning(exc_type_msg(err)) @property def metadata(self): diff --git a/src/bos/server/controllers/v2/boot_set.py b/src/bos/server/controllers/v2/boot_set.py index b12feeee..002d0be2 100644 --- a/src/bos/server/controllers/v2/boot_set.py +++ b/src/bos/server/controllers/v2/boot_set.py @@ -115,7 +115,7 @@ def validate_boot_sets(session_template: dict, except Exception as err: msg = f"Session template: '{template_name}' boot set: '{bs_name}' " \ f"could not locate its {boot_artifact}. Warning: " + exc_type_msg(err) - LOGGER.warn(msg) + LOGGER.warning(msg) warning_flag = True warn_msg = warn_msg + msg if warning_flag: diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 9d966529..8e7c80b9 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -569,7 +569,7 @@ def _populate_boot_artifacts(data): try: data['actual_state']['boot_artifacts'] = get_boot_artifacts(token) except BssTokenUnknown: - LOGGER.warn(f"Reported BSS Token: {token} is unknown.") + LOGGER.warning(f"Reported BSS Token: {token} is unknown.") return data From 45e305939860e321d36bf55e15810c40197c91a0 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:25:38 -0400 Subject: [PATCH 10/29] Linting: missing-final-newline/trailing-newline/trailing-whitespace: Tidy newlines and whitespace --- Jenkinsfile.github | 4 ++-- README.md | 4 ++-- config/uwsgi.ini | 2 +- kubernetes/cray-bos/templates/NOTES.txt | 2 +- kubernetes/cray-bos/templates/tests/test-trial.yaml | 10 +++++----- setup.cfg | 2 +- sonar-project.properties | 2 +- src/bos/common/values.py | 4 ++-- src/bos/operators/filters/filters.py | 1 - src/bos/operators/power_off_forceful.py | 1 - src/bos/operators/power_off_graceful.py | 2 -- src/bos/operators/power_on.py | 2 -- src/bos/operators/status.py | 2 -- src/bos/operators/utils/rootfs/__init__.py | 1 - src/bos/reporter/README.md | 2 +- src/bos/reporter/proc_cmdline.py | 1 - src/bos/server/controllers/v2/boot_set.py | 1 - 17 files changed, 16 insertions(+), 27 deletions(-) diff --git a/Jenkinsfile.github b/Jenkinsfile.github index e0a83efb..8b216c4f 100644 --- a/Jenkinsfile.github +++ b/Jenkinsfile.github @@ -2,7 +2,7 @@ * * MIT License * - * (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP + * (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -33,7 +33,7 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: "10")) - disableConcurrentBuilds() + disableConcurrentBuilds() timeout(time: 90, unit: 'MINUTES') timestamps() } diff --git a/README.md b/README.md index df0fa1cb..2cf5057d 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ next. Nodes consult BSS for their target artifacts and boot parameters when node boot, reboot, or initially power on. *Configuration Framework Service* (CFS) is a CRD-enabled API that launches and -aggregates status from one or more ansible instances against +aggregates status from one or more ansible instances against nodes (Node Personalization) or Images (Image Customization). ## Terminology @@ -147,7 +147,7 @@ $ ./regenerate_server.sh ## Build Helpers -This repository uses some build helper scripts from the +This repository uses some build helper scripts from the [`cms-meta-tools`](https://github.com/Cray-HPE/cms-meta-tools) repository. See that repository for more details. ## Local Builds diff --git a/config/uwsgi.ini b/config/uwsgi.ini index 75030f6e..d188849c 100644 --- a/config/uwsgi.ini +++ b/config/uwsgi.ini @@ -14,4 +14,4 @@ virtualenv=/app/venv # Added to try and help avoid OOM issues, based on # CASMTRIAGE-5369/CASMTRIAGE-6993 max-requests=1024 -harakiri=30 +harakiri=30 diff --git a/kubernetes/cray-bos/templates/NOTES.txt b/kubernetes/cray-bos/templates/NOTES.txt index abf862b9..69a09376 100644 --- a/kubernetes/cray-bos/templates/NOTES.txt +++ b/kubernetes/cray-bos/templates/NOTES.txt @@ -1,4 +1,4 @@ Installation info for chart {{ include "cray-service.name" . }}: Usage: Use the cray CLI to interface with BOS. -cray bos --help \ No newline at end of file +cray bos --help \ No newline at end of file diff --git a/kubernetes/cray-bos/templates/tests/test-trial.yaml b/kubernetes/cray-bos/templates/tests/test-trial.yaml index 64e2ea1a..b11a6e02 100644 --- a/kubernetes/cray-bos/templates/tests/test-trial.yaml +++ b/kubernetes/cray-bos/templates/tests/test-trial.yaml @@ -50,15 +50,15 @@ spec: - name: "{{ .Release.Name }}-test" image: {{ .Values.tests.image | quote }} command: - - /usr/bin/curl + - /usr/bin/curl - -v - - --connect-timeout + - --connect-timeout - "5" - - --max-time + - --max-time - "10" - - --retry + - --retry - "5" - - --retry-delay + - --retry-delay - "1" - --retry-connrefused - http://cray-bos/v2/version diff --git a/setup.cfg b/setup.cfg index 7baa30b2..6a7e3837 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ [flake8] max-line-length=110 ignore=F401,W291,W391 -exclude = +exclude = # Exclude the default controllers created by server autogeneration default_controller.py # Exclude packaging scripts diff --git a/sonar-project.properties b/sonar-project.properties index 6e2292ae..5e1af7bb 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -2,7 +2,7 @@ sonar.projectKey=CMS-bos sonar.projectName=CMS:bos sonar.projectVersion=1.0.0 -sonar.language=py +sonar.language=py sonar.python.pylint=/usr/local/bin/pylint sonar.core.codeCoveragePlugin=cobertura diff --git a/src/bos/common/values.py b/src/bos/common/values.py index 8de6a319..a0a5c8e5 100644 --- a/src/bos/common/values.py +++ b/src/bos/common/values.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -70,4 +70,4 @@ class Status: EMPTY_STAGED_STATE = { "configuration": "", "boot_artifacts": EMPTY_BOOT_ARTIFACTS -} \ No newline at end of file +} diff --git a/src/bos/operators/filters/filters.py b/src/bos/operators/filters/filters.py index e4cf1f0c..634560b9 100644 --- a/src/bos/operators/filters/filters.py +++ b/src/bos/operators/filters/filters.py @@ -288,4 +288,3 @@ def _match(self, component: dict) -> bool: if any(bool(v) for v in actual_state_boot_artifacts.values()): return True return False - diff --git a/src/bos/operators/power_off_forceful.py b/src/bos/operators/power_off_forceful.py index 93f0b0b9..e93eb86c 100644 --- a/src/bos/operators/power_off_forceful.py +++ b/src/bos/operators/power_off_forceful.py @@ -65,4 +65,3 @@ def _act(self, components): if __name__ == '__main__': main(ForcefulPowerOffOperator) - diff --git a/src/bos/operators/power_off_graceful.py b/src/bos/operators/power_off_graceful.py index c120fa4e..771ff481 100644 --- a/src/bos/operators/power_off_graceful.py +++ b/src/bos/operators/power_off_graceful.py @@ -61,5 +61,3 @@ def _act(self, components): if __name__ == '__main__': main(GracefulPowerOffOperator) - - diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index 27672a8a..76df0f13 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -148,5 +148,3 @@ def _set_bss(self, components, retries=5): if __name__ == '__main__': main(PowerOnOperator) - - diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index a82c901a..c92ca191 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -228,5 +228,3 @@ def _calculate_status(self, component, power_state, cfs_component): if __name__ == '__main__': main(StatusOperator) - - diff --git a/src/bos/operators/utils/rootfs/__init__.py b/src/bos/operators/utils/rootfs/__init__.py index 5ebf5986..7a360e75 100644 --- a/src/bos/operators/utils/rootfs/__init__.py +++ b/src/bos/operators/utils/rootfs/__init__.py @@ -93,4 +93,3 @@ def nmd_field(self): parameter. """ return None - diff --git a/src/bos/reporter/README.md b/src/bos/reporter/README.md index a3d59069..3cd1f47e 100644 --- a/src/bos/reporter/README.md +++ b/src/bos/reporter/README.md @@ -1 +1 @@ -The BOS client reports the state of the boot artifacts on the node. Specifically, it reports a Boot Artifact ID that it parses out of the kernel boot parameters found at /proc/cmdline. \ No newline at end of file +The BOS client reports the state of the boot artifacts on the node. Specifically, it reports a Boot Artifact ID that it parses out of the kernel boot parameters found at /proc/cmdline. \ No newline at end of file diff --git a/src/bos/reporter/proc_cmdline.py b/src/bos/reporter/proc_cmdline.py index 8885fbab..fd9d6f0a 100644 --- a/src/bos/reporter/proc_cmdline.py +++ b/src/bos/reporter/proc_cmdline.py @@ -61,4 +61,3 @@ def get_value_from_proc_cmdline(key): # Single string values are not interesting to us continue raise KeyError("Key '%s' was not discovered on '/proc/cmdline'" % key) - diff --git a/src/bos/server/controllers/v2/boot_set.py b/src/bos/server/controllers/v2/boot_set.py index 002d0be2..f0f8ce36 100644 --- a/src/bos/server/controllers/v2/boot_set.py +++ b/src/bos/server/controllers/v2/boot_set.py @@ -122,4 +122,3 @@ def validate_boot_sets(session_template: dict, return BOOT_SET_WARNING, warn_msg return BOOT_SET_SUCCESS, "Valid" - From 0b275e4e5231ba18f81e2133d5c2e14f9ea2e560 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 15:18:24 -0400 Subject: [PATCH 11/29] Linting: logging-not-lazy: Use lazy % formatting in logging functions --- src/bos/operators/actual_state_cleanup.py | 2 +- src/bos/operators/discovery.py | 6 +++--- src/bos/operators/utils/liveness/__main__.py | 4 ++-- src/bos/reporter/client.py | 4 ++-- src/bos/reporter/status_reporter/__main__.py | 10 +++++----- src/bos/server/controllers/utils.py | 6 +++--- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/bos/operators/actual_state_cleanup.py b/src/bos/operators/actual_state_cleanup.py index 17191e9f..241ee79d 100644 --- a/src/bos/operators/actual_state_cleanup.py +++ b/src/bos/operators/actual_state_cleanup.py @@ -65,7 +65,7 @@ def _act(self, components): 'actual_state': EMPTY_ACTUAL_STATE}) if data: LOGGER.info('Found %d components that require updates', len(data)) - LOGGER.debug('Calling to update with payload: %s' %(data)) + LOGGER.debug('Calling to update with payload: %s', data) self.bos_client.components.update_components(data) return components diff --git a/src/bos/operators/discovery.py b/src/bos/operators/discovery.py index d93f2466..2f460cf0 100644 --- a/src/bos/operators/discovery.py +++ b/src/bos/operators/discovery.py @@ -70,16 +70,16 @@ def _run(self) -> None: """ components_to_add = [] for component in sorted(self.missing_components): - LOGGER.debug("Processing new xname entity '%s'"%(component)) + LOGGER.debug("Processing new xname entity '%s'", component) new_component = copy(NEW_COMPONENT) new_component['id'] = component components_to_add.append(new_component) if not components_to_add: LOGGER.info("No new component(s) discovered.") return - LOGGER.info("%s new component(s) from HSM." %(len(components_to_add))) + LOGGER.info("%s new component(s) from HSM.", len(components_to_add)) self.bos_client.components.put_components(components_to_add) - LOGGER.info("%s new component(s) added to BOS!" %(len(components_to_add))) + LOGGER.info("%s new component(s) added to BOS!", len(components_to_add)) @property def bos_components(self) -> Set[str]: diff --git a/src/bos/operators/utils/liveness/__main__.py b/src/bos/operators/utils/liveness/__main__.py index 4fa31f91..d843afd2 100644 --- a/src/bos/operators/utils/liveness/__main__.py +++ b/src/bos/operators/utils/liveness/__main__.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -44,7 +44,7 @@ def setup_logging(): setup_logging() timestamp = Timestamp.byref(TIMESTAMP_PATH) if timestamp.alive: - LOGGER.info("%s is considered valid; the application is alive!" % (timestamp)) + LOGGER.info("%s is considered valid; the application is alive!", timestamp) sys.exit(0) else: LOGGER.warning("Timestamp is no longer considered valid.") diff --git a/src/bos/reporter/client.py b/src/bos/reporter/client.py index ddd64a3c..c6070993 100644 --- a/src/bos/reporter/client.py +++ b/src/bos/reporter/client.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -64,7 +64,7 @@ def get_auth_token(path='/opt/cray/auth-utils/bin/get-auth-token'): out = out.rstrip('\n') return out except subprocess.CalledProcessError as e: - LOGGER.error('get_auth_token failed to retrieve authorization token: code=%d: error=%s' % (e.returncode, e.output)) + LOGGER.error('get_auth_token failed to retrieve authorization token: code=%d: error=%s', e.returncode, e.output) except Exception: LOGGER.exception('Unexpected exception') LOGGER.info("Spire Token not yet available; retrying in a few seconds.") diff --git a/src/bos/reporter/status_reporter/__main__.py b/src/bos/reporter/status_reporter/__main__.py index c87ab416..3a7d2dea 100644 --- a/src/bos/reporter/status_reporter/__main__.py +++ b/src/bos/reporter/status_reporter/__main__.py @@ -73,23 +73,23 @@ def report_state_until_success(component): time_to_wait = min([backoff_ceiling, time_to_wait]) sleep(time_to_wait) attempt += 1 - LOGGER.info("Attempt %s of contacting BOS..." % (attempt)) + LOGGER.info("Attempt %s of contacting BOS...", attempt) session = requests_retry_session() try: bss_referral_token = get_value_from_proc_cmdline('bss_referral_token') state = {'bss_token': bss_referral_token} report_state(component, state, session) except UnknownComponent: - LOGGER.warning("BOS has no record of component '%s'; nothing to report." % (component)) + LOGGER.warning("BOS has no record of component '%s'; nothing to report.", component) LOGGER.warning("Will re-attempt patch operation as necessary.") continue except BOSComponentException as cce: - LOGGER.warning("Unable to contact BOS to report component status: %s" % (cce)) + LOGGER.warning("Unable to contact BOS to report component status: %s", cce) continue except OSError as exc: LOGGER.error("BOS client encountered an error: %s", exc_type_msg(exc)) continue - LOGGER.info("Updated the actual_state record for BOS component '%s'." % (component)) + LOGGER.info("Updated the actual_state record for BOS component '%s'.", component) return @@ -130,7 +130,7 @@ def main(): has_slept_before = False while True: - LOGGER.info("Attempting to report status for '%s'" % (component)) + LOGGER.info("Attempting to report status for '%s'", component) try: report_state_until_success(component) except Exception as exp: diff --git a/src/bos/server/controllers/utils.py b/src/bos/server/controllers/utils.py index 797e744b..b5aec7f3 100644 --- a/src/bos/server/controllers/utils.py +++ b/src/bos/server/controllers/utils.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2019, 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2019, 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -48,10 +48,10 @@ def url_for(endpoint, **values): Also, it always sets _external=True. """ - LOGGER.debug('url_for(endpoint=%s)' % endpoint) + LOGGER.debug('url_for(endpoint=%s)', endpoint) url = flask.url_for(endpoint, _external=True, **values) - LOGGER.debug('url_for(url=%s)' % url) + LOGGER.debug('url_for(url=%s)', url) proxy_path = os.environ.get('PROXY_PATH') if not proxy_path: From 4acd95a95603cdede53ff9d015ce0c0bb8e0347f Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 15:04:53 -0400 Subject: [PATCH 12/29] Linting: consider-using-from-import: Use from imports --- src/bos/operators/power_off_forceful.py | 2 +- src/bos/operators/power_off_graceful.py | 2 +- src/bos/operators/power_on.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bos/operators/power_off_forceful.py b/src/bos/operators/power_off_forceful.py index e93eb86c..5d258b7a 100644 --- a/src/bos/operators/power_off_forceful.py +++ b/src/bos/operators/power_off_forceful.py @@ -25,7 +25,7 @@ import logging from bos.common.values import Action, Status -import bos.operators.utils.clients.pcs as pcs +from bos.operators.utils.clients import pcs from bos.operators.utils.clients.bos.options import options from bos.operators.base import BaseOperator, main from bos.operators.filters import BOSQuery, HSMState, TimeSinceLastAction diff --git a/src/bos/operators/power_off_graceful.py b/src/bos/operators/power_off_graceful.py index 771ff481..9a925b07 100644 --- a/src/bos/operators/power_off_graceful.py +++ b/src/bos/operators/power_off_graceful.py @@ -25,7 +25,7 @@ import logging from bos.common.values import Action, Status -import bos.operators.utils.clients.pcs as pcs +from bos.operators.utils.clients import pcs from bos.operators.base import BaseOperator, main from bos.operators.filters import BOSQuery, HSMState diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index 76df0f13..50e2b450 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -28,8 +28,8 @@ from bos.common.utils import exc_type_msg from bos.common.values import Action, Status -import bos.operators.utils.clients.bss as bss -import bos.operators.utils.clients.pcs as pcs +from bos.operators.utils.clients import bss +from bos.operators.utils.clients import pcs from bos.operators.utils.clients.cfs import set_cfs from bos.operators.base import BaseOperator, main from bos.operators.filters import BOSQuery, HSMState From db5a8f27dd30f0567b0c59c4b0e93b5cbca9dcd3 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 14:30:28 -0400 Subject: [PATCH 13/29] Linting: useless-import-alias: Remove unnecessary import alias --- src/bos/server/controllers/v2/healthz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bos/server/controllers/v2/healthz.py b/src/bos/server/controllers/v2/healthz.py index 797eadbc..166c0aea 100644 --- a/src/bos/server/controllers/v2/healthz.py +++ b/src/bos/server/controllers/v2/healthz.py @@ -24,7 +24,7 @@ import logging from bos.common.utils import exc_type_msg -from bos.server.models.healthz import Healthz as Healthz +from bos.server.models.healthz import Healthz from bos.server import redis_db_utils DB = redis_db_utils.get_wrapper(db='options') From 5a0a645067fff3d2e50938f3147698f5a1ea9f40 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 11 Jul 2024 13:10:29 -0400 Subject: [PATCH 14/29] Linting: unused-import: Remove unused imports --- src/bos/operators/configuration.py | 4 +--- src/bos/operators/discovery.py | 6 +++--- src/bos/operators/utils/clients/cfs.py | 2 +- src/bos/server/controllers/v2/base.py | 3 --- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/bos/operators/configuration.py b/src/bos/operators/configuration.py index decef243..fa6f1975 100644 --- a/src/bos/operators/configuration.py +++ b/src/bos/operators/configuration.py @@ -22,14 +22,12 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -from collections import defaultdict import logging from bos.common.values import Status from bos.operators.utils.clients.cfs import set_cfs from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, DesiredConfigurationIsNone, DesiredConfigurationSetInCFS, \ - BootArtifactStatesMatch, NOT +from bos.operators.filters import BOSQuery, DesiredConfigurationSetInCFS, NOT LOGGER = logging.getLogger('bos.operators.configuration') diff --git a/src/bos/operators/discovery.py b/src/bos/operators/discovery.py index 2f460cf0..6f5ec77a 100644 --- a/src/bos/operators/discovery.py +++ b/src/bos/operators/discovery.py @@ -2,7 +2,7 @@ # # MIT License # -# (C) Copyright 2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -27,8 +27,8 @@ from copy import copy from bos.common.values import Action, EMPTY_ACTUAL_STATE, EMPTY_DESIRED_STATE -from bos.operators.utils.clients.hsm import read_all_node_xnames, HWStateManagerException -from bos.operators.base import BaseOperator, main, _update_log_level +from bos.operators.utils.clients.hsm import read_all_node_xnames +from bos.operators.base import BaseOperator, main LOGGER = logging.getLogger(__name__) diff --git a/src/bos/operators/utils/clients/cfs.py b/src/bos/operators/utils/clients/cfs.py index 0af1b2e3..44c040bb 100644 --- a/src/bos/operators/utils/clients/cfs.py +++ b/src/bos/operators/utils/clients/cfs.py @@ -23,7 +23,7 @@ # from collections import defaultdict import logging -from requests.exceptions import HTTPError, ConnectionError +from requests.exceptions import HTTPError from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL diff --git a/src/bos/server/controllers/v2/base.py b/src/bos/server/controllers/v2/base.py index 7b0d54fc..8ff927bc 100644 --- a/src/bos/server/controllers/v2/base.py +++ b/src/bos/server/controllers/v2/base.py @@ -21,15 +21,12 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -import os.path import logging -import subprocess import yaml from bos.common.utils import exc_type_msg from bos.server.controllers.utils import url_for from bos.server.models import Version, Link -from os import path LOGGER = logging.getLogger('bos.server.controllers.v2.base') From e749e83432a6584a6211ac00082b11b29f280af3 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:35:58 -0400 Subject: [PATCH 15/29] Linting: wrong-import-order: Standard imports, then third party, then local --- src/bos/common/tenant_utils.py | 3 ++- src/bos/operators/session_setup.py | 3 +-- src/bos/operators/utils/clients/bos/sessions_status.py | 2 +- src/bos/operators/utils/clients/bss.py | 3 ++- src/bos/operators/utils/clients/pcs.py | 3 ++- src/bos/operators/utils/clients/s3.py | 2 +- src/bos/reporter/client.py | 9 +++++---- src/bos/server/controllers/base.py | 2 +- src/bos/server/controllers/utils.py | 2 +- src/bos/server/controllers/v2/components.py | 3 ++- src/bos/server/controllers/v2/options.py | 3 ++- src/bos/server/controllers/v2/sessions.py | 3 ++- src/bos/server/redis_db_utils.py | 3 ++- 13 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/bos/common/tenant_utils.py b/src/bos/common/tenant_utils.py index 0fb9c47c..db06921a 100644 --- a/src/bos/common/tenant_utils.py +++ b/src/bos/common/tenant_utils.py @@ -22,10 +22,11 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import connexion import functools import logging import hashlib + +import connexion from requests.exceptions import HTTPError from bos.common.utils import exc_type_msg, requests_retry_session, PROTOCOL diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 46d4891f..e909cabc 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -24,9 +24,8 @@ # import copy import logging -from botocore.exceptions import ClientError from typing import Set - +from botocore.exceptions import ClientError from bos.operators.base import BaseOperator, main from bos.operators.filters.filters import HSMState diff --git a/src/bos/operators/utils/clients/bos/sessions_status.py b/src/bos/operators/utils/clients/bos/sessions_status.py index 1400e205..36a83a91 100644 --- a/src/bos/operators/utils/clients/bos/sessions_status.py +++ b/src/bos/operators/utils/clients/bos/sessions_status.py @@ -24,9 +24,9 @@ import json import logging -from .base import BASE_ENDPOINT, log_call_errors from bos.common.tenant_utils import get_new_tenant_header from bos.common.utils import requests_retry_session +from .base import BASE_ENDPOINT, log_call_errors LOGGER = logging.getLogger('bos.operators.utils.clients.bos.sessions_status') diff --git a/src/bos/operators/utils/clients/bss.py b/src/bos/operators/utils/clients/bss.py index 885a8639..8cc94d5e 100644 --- a/src/bos/operators/utils/clients/bss.py +++ b/src/bos/operators/utils/clients/bss.py @@ -21,10 +21,11 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -from requests.exceptions import HTTPError import logging import json +from requests.exceptions import HTTPError + from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL LOGGER = logging.getLogger(__name__) diff --git a/src/bos/operators/utils/clients/pcs.py b/src/bos/operators/utils/clients/pcs.py index bebe2703..75f9b4fc 100644 --- a/src/bos/operators/utils/clients/pcs.py +++ b/src/bos/operators/utils/clients/pcs.py @@ -26,10 +26,11 @@ # https://github.com/Cray-HPE/hms-power-control/blob/develop/api/swagger.yaml import logging -import requests import json from collections import defaultdict +import requests + from bos.common.utils import compact_response_text, requests_retry_session, PROTOCOL SERVICE_NAME = 'cray-power-control' diff --git a/src/bos/operators/utils/clients/s3.py b/src/bos/operators/utils/clients/s3.py index 2378bd32..c093f27e 100644 --- a/src/bos/operators/utils/clients/s3.py +++ b/src/bos/operators/utils/clients/s3.py @@ -25,11 +25,11 @@ import logging import os import threading +from urllib.parse import urlparse import boto3 from botocore.exceptions import ClientError, ParamValidationError from botocore.config import Config as BotoConfig -from urllib.parse import urlparse from bos.common.utils import exc_type_msg diff --git a/src/bos/reporter/client.py b/src/bos/reporter/client.py index c6070993..4281953e 100644 --- a/src/bos/reporter/client.py +++ b/src/bos/reporter/client.py @@ -27,14 +27,15 @@ """ import os -from . import PROTOCOL - import logging +import subprocess +import time + import requests from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry -import subprocess -import time + +from . import PROTOCOL LOGGER = logging.getLogger('bos.reporter.client') diff --git a/src/bos/server/controllers/base.py b/src/bos/server/controllers/base.py index bf5e8274..11cf66a2 100644 --- a/src/bos/server/controllers/base.py +++ b/src/bos/server/controllers/base.py @@ -23,10 +23,10 @@ # # Cray-provided base controllers for the Boot Orchestration Service +import logging from bos.server.controllers.v2 import base as v2_base -import logging LOGGER = logging.getLogger('bos.server.controllers.base') diff --git a/src/bos/server/controllers/utils.py b/src/bos/server/controllers/utils.py index b5aec7f3..501f394e 100644 --- a/src/bos/server/controllers/utils.py +++ b/src/bos/server/controllers/utils.py @@ -21,13 +21,13 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +import logging import os from urllib.parse import urlparse, urlunparse import connexion import flask -import logging LOGGER = logging.getLogger('bos.server.controllers.utils') diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 8e7c80b9..f3421b3c 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -21,9 +21,10 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -import connexion import logging +import connexion + from bos.common.utils import exc_type_msg, get_current_timestamp from bos.common.tenant_utils import get_tenant_from_header, get_tenant_component_set, tenant_error_handler, get_tenant_aware_key from bos.common.values import Phase, Action, Status, EMPTY_STAGED_STATE, EMPTY_BOOT_ARTIFACTS diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index 459812eb..f77e0977 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -22,10 +22,11 @@ # OTHER DEALINGS IN THE SOFTWARE. # import logging -import connexion import threading import time +import connexion + from bos.common.utils import exc_type_msg from bos.server import redis_db_utils as dbutils from bos.server.models.v2_options import V2Options as Options diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index 7af9ef6e..178ee765 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -21,12 +21,13 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -import connexion from datetime import timedelta from collections import defaultdict, Counter import re import logging import uuid + +import connexion from connexion.lifecycle import ConnexionResponse from bos.common.tenant_utils import get_tenant_from_header, get_tenant_aware_key, reject_invalid_tenant diff --git a/src/bos/server/redis_db_utils.py b/src/bos/server/redis_db_utils.py index 21f4fdb5..ee1d83f0 100644 --- a/src/bos/server/redis_db_utils.py +++ b/src/bos/server/redis_db_utils.py @@ -21,10 +21,11 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # -import connexion import functools import json import logging + +import connexion import redis from bos.common.utils import exc_type_msg From f7cc47e15669460aef6bf9c4b8a126bff66f26b7 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 11:58:00 -0400 Subject: [PATCH 16/29] Linting: logging-format-interpolation: Use lazy % formatting in logging functions --- src/bos/operators/base.py | 16 ++++++++-------- src/bos/operators/filters/base.py | 14 +++++--------- src/bos/operators/power_on.py | 2 +- src/bos/operators/session_completion.py | 4 ++-- src/bos/operators/status.py | 4 ++-- src/bos/operators/utils/clients/bss.py | 2 +- src/bos/operators/utils/clients/hsm.py | 2 +- src/bos/operators/utils/rootfs/factory.py | 2 +- src/bos/server/controllers/v2/options.py | 8 ++++---- src/bos/server/controllers/v2/sessions.py | 4 ++-- src/bos/server/redis_db_utils.py | 2 +- 11 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index bb121c16..2f890e05 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -98,14 +98,14 @@ def run(self) -> NoReturn: _update_log_level() self._run() except Exception as e: - LOGGER.exception('Unhandled exception detected: {}'.format(e)) + LOGGER.exception('Unhandled exception detected: %s', e) try: sleep_time = getattr(options, self.frequency_option) - (time.time() - start_time) if sleep_time > 0: time.sleep(sleep_time) except Exception as e: - LOGGER.exception('Unhandled exception getting polling frequency: {}'.format(e)) + LOGGER.exception('Unhandled exception getting polling frequency: %s', e) time.sleep(5) # A small sleep for when exceptions getting the polling frequency def _run(self) -> None: @@ -125,8 +125,8 @@ def _run(self) -> None: try: components = self._act(components) except Exception as e: - LOGGER.error("An unhandled exception was caught while trying to act on components: {}".format(e), - exec_info=True) + LOGGER.error("An unhandled exception was caught while trying to act on components: %s", + e, exec_info=True) for component in components: component["error"] = str(e) self._update_database(components) @@ -259,12 +259,12 @@ def _update_log_level() -> None: new_level = logging.getLevelName(options.logging_level.upper()) current_level = LOGGER.getEffectiveLevel() if current_level != new_level: - LOGGER.log(current_level, 'Changing logging level from {} to {}'.format( - logging.getLevelName(current_level), logging.getLevelName(new_level))) + LOGGER.log(current_level, 'Changing logging level from %s to %s', + logging.getLevelName(current_level), new_level) logger = logging.getLogger() logger.setLevel(new_level) - LOGGER.log(new_level, 'Logging level changed from {} to {}'.format( - logging.getLevelName(current_level), logging.getLevelName(new_level))) + LOGGER.log(new_level, 'Logging level changed from %s to %s', + logging.getLevelName(current_level), new_level) except Exception as e: LOGGER.error('Error updating logging level: %s', exc_type_msg(e)) diff --git a/src/bos/operators/filters/base.py b/src/bos/operators/filters/base.py index fa21ece9..5f03c236 100644 --- a/src/bos/operators/filters/base.py +++ b/src/bos/operators/filters/base.py @@ -2,7 +2,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -60,10 +60,8 @@ class IDFilter(BaseFilter, ABC): def filter(self, components: List[dict]) -> List[dict]: component_ids = [component['id'] for component in components] results = BaseFilter.filter(self, components=component_ids) - LOGGER.debug('{} filter found the following components: {}'.format( - type(self).__name__, - ','.join(results) - )) + LOGGER.debug('%s filter found the following components: %s', type(self).__name__, + ','.join(results)) return [component for component in components if component['id'] in results] @@ -71,10 +69,8 @@ class DetailsFilter(BaseFilter, ABC): """ A class for filters that take and return lists of detailed component information """ def filter(self, components: List[dict]) -> List[dict]: results = BaseFilter.filter(self, components=components) - LOGGER.debug('{} filter found the following components: {}'.format( - type(self).__name__, - ','.join([component.get('id', '') for component in results]) - )) + LOGGER.debug('%s filter found the following components: %s', type(self).__name__, + ','.join([component.get('id', '') for component in results])) return results diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index 50e2b450..bec0f605 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -143,7 +143,7 @@ def _set_bss(self, components, retries=5): "session": comp["session"] } for comp in bss_tokens ] - LOGGER.debug('Updated components (minus desired_state data): {}'.format(redacted_component_updates)) + LOGGER.debug('Updated components (minus desired_state data): %s', redacted_component_updates) self.bos_client.components.update_components(bss_tokens) if __name__ == '__main__': diff --git a/src/bos/operators/session_completion.py b/src/bos/operators/session_completion.py index 29132719..32f4f28f 100644 --- a/src/bos/operators/session_completion.py +++ b/src/bos/operators/session_completion.py @@ -2,7 +2,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -70,7 +70,7 @@ def _mark_session_complete(self, session_id, tenant): 'end_time': get_current_timestamp()}}) # This call causes the session status to saved in the database. self.bos_client.session_status.post_session_status(session_id, tenant) - LOGGER.info('Session {} is complete'.format(session_id)) + LOGGER.info('Session %s is complete', session_id) if __name__ == '__main__': diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index c92ca191..e9f70589 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -87,8 +87,8 @@ def _run(self) -> None: if not updated_components: LOGGER.debug('No components require status updates') return - LOGGER.info('Found {} components that require status updates'.format(len(updated_components))) - LOGGER.debug(f'Updated components: {updated_components}') + LOGGER.info('Found %d components that require status updates', len(updated_components)) + LOGGER.debug('Updated components: %s', updated_components) self.bos_client.components.update_components(updated_components) @staticmethod diff --git a/src/bos/operators/utils/clients/bss.py b/src/bos/operators/utils/clients/bss.py index 8cc94d5e..d3e7b13a 100644 --- a/src/bos/operators/utils/clients/bss.py +++ b/src/bos/operators/utils/clients/bss.py @@ -66,7 +66,7 @@ def set_bss(node_set, kernel_params, kernel, initrd, session=None): raise Exception("set_bss called with empty node_set") session = session or requests_retry_session() - LOGGER.info("Params: {}".format(kernel_params)) + LOGGER.info("Params: %s", kernel_params) url = "%s/bootparameters" % (ENDPOINT) # Assignment payload diff --git a/src/bos/operators/utils/clients/hsm.py b/src/bos/operators/utils/clients/hsm.py index 5b5b696a..39569e22 100644 --- a/src/bos/operators/utils/clients/hsm.py +++ b/src/bos/operators/utils/clients/hsm.py @@ -235,5 +235,5 @@ def get(self, path, params=None): try: return response.json() except ValueError: - LOGGER.error("Couldn't parse a JSON response: {}".format(response.text)) + LOGGER.error("Couldn't parse a JSON response: %s", response.text) raise diff --git a/src/bos/operators/utils/rootfs/factory.py b/src/bos/operators/utils/rootfs/factory.py index 442bbe05..86a3064b 100644 --- a/src/bos/operators/utils/rootfs/factory.py +++ b/src/bos/operators/utils/rootfs/factory.py @@ -61,7 +61,7 @@ def __call__(self): module = importlib.import_module(provider_module) except ModuleNotFoundError as mnfe: # This is pretty much unrecoverable at this stage of development; make note and raise - LOGGER.error("Provider provisioning mechanism '{}' not yet implemented or not found.".format(provider_name)) + LOGGER.error("Provider provisioning mechanism '%s' not yet implemented or not found.", provider_name) raise ProviderNotImplemented(mnfe) from mnfe class_def = getattr(module, provider_classname) diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index f77e0977..fab768b1 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -131,12 +131,12 @@ def update_log_level(new_level_str): new_level = logging.getLevelName(new_level_str.upper()) current_level = LOGGER.getEffectiveLevel() if current_level != new_level: - LOGGER.log(current_level, 'Changing logging level from {} to {}'.format( - logging.getLevelName(current_level), logging.getLevelName(new_level))) + LOGGER.log(current_level, 'Changing logging level from %s to %s', + logging.getLevelName(current_level), new_level) logger = logging.getLogger() logger.setLevel(new_level) - LOGGER.log(new_level, 'Logging level changed from {} to {}'.format( - logging.getLevelName(current_level), logging.getLevelName(new_level))) + LOGGER.log(new_level, 'Logging level changed from %s to %s', + logging.getLevelName(current_level), new_level) def check_v2_logging_level(): diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index 178ee765..b0ee2946 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -284,13 +284,13 @@ def _get_filtered_sessions(tenant, min_age, max_age, status): try: max_start = _age_to_timestamp(min_age) except Exception as e: - LOGGER.warning('Unable to parse age: {}'.format(min_age)) + LOGGER.warning('Unable to parse min_age: %s', min_age) raise ParsingException(e) from e if max_age: try: min_start = _age_to_timestamp(max_age) except Exception as e: - LOGGER.warning('Unable to parse age: {}'.format(max_age)) + LOGGER.warning('Unable to parse max_age: %s', max_age) raise ParsingException(e) from e if any([min_start, max_start, status, tenant]): response = [r for r in response if _matches_filter(r, tenant, min_start, max_start, status)] diff --git a/src/bos/server/redis_db_utils.py b/src/bos/server/redis_db_utils.py index ee1d83f0..4470fb7a 100644 --- a/src/bos/server/redis_db_utils.py +++ b/src/bos/server/redis_db_utils.py @@ -149,7 +149,7 @@ def wrapper(*args, **kwargs): del kwargs['body'] return func(*args, **kwargs) except redis.exceptions.ConnectionError as e: - LOGGER.error('Unable to connect to the Redis database: {}'.format(e)) + LOGGER.error('Unable to connect to the Redis database: %s', e) return connexion.problem( status=503, title='Unable to connect to the Redis database', detail=str(e)) From 82941b937cd01e716eb6cab00ab7622ef5d89795 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 12:17:23 -0400 Subject: [PATCH 17/29] Linting: logging-fstring-interpolation: Use lazy % formatting in logging functions --- src/bos/operators/base.py | 6 ++-- src/bos/server/controllers/v2/components.py | 40 ++++++++++----------- src/bos/server/controllers/v2/sessions.py | 2 +- src/bos/server/dbs/boot_artifacts.py | 6 ++-- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 2f890e05..911cc985 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -200,7 +200,7 @@ def _update_database(self, components: List[dict], additional_fields: dict=None) raise MissingSessionData data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) - LOGGER.debug(f'Updated components: {data}') + LOGGER.debug('Updated components: %s', data) self.bos_client.components.update_components(data) def _preset_last_action(self, components: List[dict]) -> None: @@ -226,7 +226,7 @@ def _preset_last_action(self, components: List[dict]) -> None: patch['last_action'] = last_action_data data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) - LOGGER.debug(f'Updated components: {data}') + LOGGER.debug('Updated components: %s', data) self.bos_client.components.update_components(data) def _update_database_for_failure(self, components: List[dict]) -> None: @@ -247,7 +247,7 @@ def _update_database_for_failure(self, components: List[dict]) -> None: patch['error'] = 'The retry limit has been hit for this component, but no services have reported specific errors' data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) - LOGGER.debug(f'Updated components: {data}') + LOGGER.debug('Updated components: %s', data) self.bos_client.components.update_components(data) diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index f3421b3c..2c848f03 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -111,36 +111,34 @@ def _calculate_status(data): Calculates and returns the status of a component """ if not 'status' in data: - LOGGER.debug(f"No status in data: {data}. This will have the effect of clearing any pre-existing phase.") + LOGGER.debug("No status in data: %s. This will have the effect of clearing any pre-existing phase.", data) status_data = data.get('status', {}) override = status_data.get('status_override', '') if override: return override phase = status_data.get('phase', '') - last_action = data.get('last_action', {}).get('action', '') component = data.get('id', '') - now = get_current_timestamp() + last_action = data.get('last_action', {}).get('action', '') + + status = status = Status.stable if phase == Phase.powering_on: if last_action == Action.power_on and not data.get('last_action', {}).get('failed', False): - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_on_called}") - return Status.power_on_called - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_on_pending}") - return Status.power_on_pending - if phase == Phase.powering_off: + status = Status.power_on_called + else: + status = Status.power_on_pending + elif phase == Phase.powering_off: if last_action == Action.power_off_gracefully: - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_gracefully_called}") - return Status.power_off_gracefully_called - if last_action == Action.power_off_forcefully: - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_forcefully_called}") - return Status.power_off_forcefully_called - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.power_off_pending}") - return Status.power_off_pending - if phase == Phase.configuring: - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.configuring}") - return Status.configuring - LOGGER.debug(f"{now} Component: {component} Phase: {phase} Status: {Status.stable}") - return Status.stable + status = Status.power_off_gracefully_called + elif last_action == Action.power_off_forcefully: + status = Status.power_off_forcefully_called + else: + status = Status.power_off_pending + elif phase == Phase.configuring: + status = Status.configuring + + LOGGER.debug("Component: %s Last action: %s Phase: %s Status: %s", component, last_action, phase, status) + return status def _matches_filter(data, enabled, session, staged_session, phase, status): @@ -570,7 +568,7 @@ def _populate_boot_artifacts(data): try: data['actual_state']['boot_artifacts'] = get_boot_artifacts(token) except BssTokenUnknown: - LOGGER.warning(f"Reported BSS Token: {token} is unknown.") + LOGGER.warning("Reported BSS Token '%s' is unknown.", token) return data diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index b0ee2946..f2356b3c 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -70,7 +70,7 @@ def post_v2_session(): # noqa: E501 LOGGER.error(msg) return msg, 400 template_name = session_create.template_name - LOGGER.debug(f"Template Name: {template_name} operation: {session_create.operation}") + LOGGER.debug("Template Name: %s operation: %s", template_name, session_create.operation) # Check that the template_name exists. session_template_response = get_v2_sessiontemplate(template_name) if isinstance(session_template_response, ConnexionResponse): diff --git a/src/bos/server/dbs/boot_artifacts.py b/src/bos/server/dbs/boot_artifacts.py index ffb3f6cd..80f02aad 100644 --- a/src/bos/server/dbs/boot_artifacts.py +++ b/src/bos/server/dbs/boot_artifacts.py @@ -48,10 +48,8 @@ def record_boot_artifacts(token: str, Associate the BSS token with the boot artifacts. BSS returns a token after BOS asks it to create or update the boot artifacts. """ - LOGGER.info(f"Logging BSS token: {token} and boot artifacts: " - f"\nkernel: {kernel}" - f"\nkernel_parameters: {kernel_parameters}" - f"\ninitrd: {initrd}") + LOGGER.info("Logging BSS token and boot artifacts: token='%s' kernel='%s' " + "kernel_parameters='%s' initrd='%s'", token, kernel, kernel_parameters, initrd) TOKENS_DB.put(token, {"kernel": kernel, "kernel_parameters": kernel_parameters, "initrd": initrd, From 9853b6302f58c34fbf46e1618dc4132b8ec3a525 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 12 Jul 2024 13:20:55 -0400 Subject: [PATCH 18/29] Linting: line-too-long --- src/bos/common/utils.py | 13 ++-- src/bos/operators/actual_state_cleanup.py | 4 +- src/bos/operators/base.py | 20 +++-- src/bos/operators/filters/filters.py | 20 +++-- src/bos/operators/power_on.py | 9 ++- src/bos/operators/session_cleanup.py | 7 +- src/bos/operators/session_completion.py | 6 +- src/bos/operators/session_setup.py | 77 +++++++++++-------- src/bos/operators/status.py | 41 ++++++---- .../s3_boot_image_metadata.py | 6 +- src/bos/operators/utils/clients/bos/base.py | 3 +- .../operators/utils/clients/bos/options.py | 3 +- src/bos/operators/utils/clients/pcs.py | 59 +++++++++----- src/bos/operators/utils/rootfs/factory.py | 9 ++- src/bos/reporter/client.py | 4 +- src/bos/reporter/components/state.py | 6 +- src/bos/reporter/node_identity.py | 3 +- src/bos/server/controllers/v2/boot_set.py | 3 +- src/bos/server/controllers/v2/components.py | 57 +++++++++----- src/bos/server/controllers/v2/sessions.py | 33 +++++--- .../server/controllers/v2/sessiontemplates.py | 14 ++-- src/bos/server/redis_db_utils.py | 3 +- 22 files changed, 258 insertions(+), 142 deletions(-) diff --git a/src/bos/common/utils.py b/src/bos/common/utils.py index e7bb85f0..c134a528 100644 --- a/src/bos/common/utils.py +++ b/src/bos/common/utils.py @@ -63,9 +63,9 @@ def duration_to_timedelta(timestamp: str): class TimeoutHTTPAdapter(HTTPAdapter): """ - An HTTP Adapter that allows a session level timeout for both read and connect attributes. This prevents interruption - to reads that happen as a function of time or istio resets that causes our applications to sit and wait forever on - a half open socket. + An HTTP Adapter that allows a session level timeout for both read and connect attributes. + This prevents interruption to reads that happen as a function of time or istio resets that + causes our applications to sit and wait forever on a half open socket. """ def __init__(self, *args, **kwargs): if "timeout" in kwargs: @@ -102,8 +102,8 @@ def requests_retry_session(retries=10, backoff_factor=0.5, def compact_response_text(response_text: str) -> str: """ Often JSON is "pretty printed" in response text, which is undesirable for our logging. - This function transforms the response text into a single line, stripping leading and trailing whitespace from each line, - and then returns it. + This function transforms the response text into a single line, stripping leading and + trailing whitespace from each line, and then returns it. """ if response_text: return ' '.join([ line.strip() for line in response_text.split('\n') ]) @@ -112,6 +112,7 @@ def compact_response_text(response_text: str) -> str: def exc_type_msg(exc: Exception) -> str: """ - Given an exception, returns a string of its type and its text (e.g. TypeError: 'int' object is not subscriptable) + Given an exception, returns a string of its type and its text + (e.g. TypeError: 'int' object is not subscriptable) """ return ''.join(traceback.format_exception_only(type(exc), exc)) diff --git a/src/bos/operators/actual_state_cleanup.py b/src/bos/operators/actual_state_cleanup.py index 241ee79d..b6c92ba2 100644 --- a/src/bos/operators/actual_state_cleanup.py +++ b/src/bos/operators/actual_state_cleanup.py @@ -55,7 +55,9 @@ def filters(self): return [ BOSQuery(), ActualBootStateIsSet(), - ActualStateAge(seconds=duration_to_timedelta(options.component_actual_state_ttl).total_seconds()) + ActualStateAge( + seconds=duration_to_timedelta(options.component_actual_state_ttl).total_seconds() + ) ] def _act(self, components): diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 911cc985..248dbf6f 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -115,10 +115,12 @@ def _run(self) -> None: LOGGER.debug('Found 0 components that require action') return LOGGER.info('Found %d components that require action', len(components)) - if self.retry_attempt_field: # Only check for failed components if we track retries for this operator + # Only check for failed components if we track retries for this operator + if self.retry_attempt_field: components = self._handle_failed_components(components) if not components: - LOGGER.debug('After removing components that exceeded their retry limit, 0 components require action') + LOGGER.debug('After removing components that exceeded their retry limit, 0 ' + 'components require action') return for component in components: # Unset old errors components component['error'] = '' @@ -185,7 +187,10 @@ def _update_database(self, components: List[dict], additional_fields: dict=None) patch['last_action'] = last_action_data if self.retry_attempt_field: event_stats_data = { - self.retry_attempt_field: component.get('event_stats', {}).get(self.retry_attempt_field, 0) + 1 + self.retry_attempt_field: component.get( + 'event_stats', + {} + ).get(self.retry_attempt_field, 0) + 1 } patch['event_stats'] = event_stats_data @@ -204,8 +209,10 @@ def _update_database(self, components: List[dict], additional_fields: dict=None) self.bos_client.components.update_components(data) def _preset_last_action(self, components: List[dict]) -> None: - # This is done to eliminate the window between performing an action and marking the nodes as acted - # e.g. nodes could be powered-on without the correct power-on last action, causing status problems + # This is done to eliminate the window between performing an action and marking the + # nodes as acted + # e.g. nodes could be powered-on without the correct power-on last action, causing + # status problems if not self.name: return if not components: @@ -244,7 +251,8 @@ def _update_database_for_failure(self, components: List[dict]) -> None: 'status': {'status_override': Status.failed} } if not component['error']: - patch['error'] = 'The retry limit has been hit for this component, but no services have reported specific errors' + patch['error'] = ('The retry limit has been hit for this component, ' + 'but no services have reported specific errors') data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) diff --git a/src/bos/operators/filters/filters.py b/src/bos/operators/filters/filters.py index 634560b9..fd6e0f6f 100644 --- a/src/bos/operators/filters/filters.py +++ b/src/bos/operators/filters/filters.py @@ -31,7 +31,8 @@ from bos.common.utils import get_current_time, load_timestamp from bos.operators.filters.base import BaseFilter, DetailsFilter, IDFilter, LocalFilter from bos.operators.utils.clients.bos import BOSClient -from bos.operators.utils.clients.cfs import get_components_from_id_list as get_cfs_components_from_id_list +from bos.operators.utils.clients.cfs import get_components_from_id_list as \ + get_cfs_components_from_id_list from bos.operators.utils.clients.hsm import get_components as get_hsm_components LOGGER = logging.getLogger('bos.operators.filters.filters') @@ -92,9 +93,10 @@ def _filter(self, components: List[str]) -> List[str]: def filter_by_arch(self, nodes, arch): """ - Given a list of component names, query HSM for state information pertaining to arch. Components that match one - of the arch values specified are returned as a list of component IDs. HSM components that do not have arch - information are considered to be of type 'Unknown' for reasons of compatibility. + Given a list of component names, query HSM for state information pertaining to arch. + Components that match one of the arch values specified are returned as a list of + component IDs. HSM components that do not have arch information are considered to be + of type 'Unknown' for reasons of compatibility. args: components: a set of xnames arch: a set containing HSM archs as represented by strings @@ -135,8 +137,10 @@ def __init__(self, **kwargs) -> None: def _match(self, component: dict) -> bool: last_action_time = component.get('last_action', {}).get('last_updated') + if not last_action_time: + return True now = get_current_time() - if not last_action_time or now > load_timestamp(last_action_time) + timedelta(**self.kwargs): + if now > load_timestamp(last_action_time) + timedelta(**self.kwargs): return True return False @@ -167,8 +171,10 @@ def _match(self, component: dict) -> bool: if desired_boot_state.get(key, None) != actual_boot_state.get(key, None): return False # Filter out kernel parameters that dynamically change. - actual_kernel_parameters = self._sanitize_kernel_parameters(actual_boot_state.get('kernel_parameters', None)) - desired_kernel_parameters = self._sanitize_kernel_parameters(desired_boot_state.get('kernel_parameters', None)) + actual_kernel_parameters = self._sanitize_kernel_parameters( + actual_boot_state.get('kernel_parameters', None)) + desired_kernel_parameters = self._sanitize_kernel_parameters( + desired_boot_state.get('kernel_parameters', None)) if actual_kernel_parameters != desired_kernel_parameters: return False diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index bec0f605..93871f2f 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -66,16 +66,16 @@ def _act(self, components): try: self._set_bss(components) except Exception as e: - raise Exception("An error was encountered while setting BSS information: {}".format(e)) from e + raise Exception(f"Error encountered setting BSS information: {e}") from e try: set_cfs(components, enabled=False, clear_state=True) except Exception as e: - raise Exception("An error was encountered while setting CFS information: {}".format(e)) from e + raise Exception(f"Error encountered setting CFS information: {e}") from e component_ids = [component['id'] for component in components] try: pcs.power_on(component_ids) except Exception as e: - raise Exception("An error was encountered while calling CAPMC to power on: {}".format(e)) from e + raise Exception(f"Error encountered calling CAPMC to power on: {e}") from e return components def _set_bss(self, components, retries=5): @@ -143,7 +143,8 @@ def _set_bss(self, components, retries=5): "session": comp["session"] } for comp in bss_tokens ] - LOGGER.debug('Updated components (minus desired_state data): %s', redacted_component_updates) + LOGGER.debug('Updated components (minus desired_state data): %s', + redacted_component_updates) self.bos_client.components.update_components(bss_tokens) if __name__ == '__main__': diff --git a/src/bos/operators/session_cleanup.py b/src/bos/operators/session_cleanup.py index a0c304c7..bf26ca64 100644 --- a/src/bos/operators/session_cleanup.py +++ b/src/bos/operators/session_cleanup.py @@ -2,7 +2,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -66,8 +66,9 @@ def _run(self) -> None: if self.disabled: return - self.bos_client.sessions.delete_sessions(**{'status': 'complete', - 'min_age': options.cleanup_completed_session_ttl}) + self.bos_client.sessions.delete_sessions( + **{'status': 'complete', + 'min_age': options.cleanup_completed_session_ttl}) if __name__ == '__main__': main(SessionCleanupOperator) diff --git a/src/bos/operators/session_completion.py b/src/bos/operators/session_completion.py index 32f4f28f..56076b36 100644 --- a/src/bos/operators/session_completion.py +++ b/src/bos/operators/session_completion.py @@ -66,8 +66,10 @@ def _get_incomplete_components(self, session_id): return components def _mark_session_complete(self, session_id, tenant): - self.bos_client.sessions.update_session(session_id, tenant, {'status': {'status': 'complete', - 'end_time': get_current_timestamp()}}) + self.bos_client.sessions.update_session(session_id, tenant, + { 'status': { 'status': 'complete', + 'end_time': get_current_timestamp() + }}) # This call causes the session status to saved in the database. self.bos_client.session_status.post_session_status(session_id, tenant) LOGGER.info('Session %s is complete', session_id) diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index e909cabc..2eeca39e 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -104,7 +104,8 @@ def operation_type(self): def template(self): if not self._template: template_name = self.session_data.get('template_name') - self._template = self.bos_client.session_templates.get_session_template(template_name, self.tenant) + self._template = self.bos_client.session_templates.get_session_template(template_name, + self.tenant) return self._template def setup(self): @@ -166,11 +167,13 @@ def _get_boot_set_component_list(self, boot_set) -> Set[str]: continue nodes |= self.inventory.roles[role_name] if not nodes: - self._log(LOGGER.warning, "After populating node list, before any filtering, no nodes to act upon.") + self._log(LOGGER.warning, + "After populating node list, before any filtering, no nodes to act upon.") return nodes - self._log(LOGGER.debug, "Before any limiting or filtering, %d nodes to act upon.", len(nodes)) - # Filter out any nodes that do not match the boot set architecture desired; boot sets that do not have a - # specified arch are considered 'X86' nodes. + self._log(LOGGER.debug, "Before any limiting or filtering, %d nodes to act upon.", + len(nodes)) + # Filter out any nodes that do not match the boot set architecture desired; boot sets that + # do not have a specified arch are considered 'X86' nodes. arch = boot_set.get('arch', 'X86') nodes = self._apply_arch(nodes, arch) if not nodes: @@ -189,14 +192,14 @@ def _get_boot_set_component_list(self, boot_set) -> Set[str]: def _apply_arch(self, nodes, arch): """ - Removes any node from that does not match arch. Nodes in HSM that do not have the arch field, and nodes - that have the arch field flagged as undefined are assumed to be of type 'X86'. String value of arch directly - corresponds to those values in HSM components; this string is case-sensitive ('ARM' works, 'arm' does not). - Similarly, we cannot query HSM using an all-caps approach because of 'Other' and 'Unknown' would then never - match. - - Because nodes may not have a known architecture, all nodes that are of unknown architecture count as being of - type X86. + Removes any node from that does not match arch. Nodes in HSM that do not have the + arch field, and nodes that have the arch field flagged as undefined are assumed to be of + type 'X86'. String value of arch directly corresponds to those values in HSM components; + this string is case-sensitive ('ARM' works, 'arm' does not). Similarly, we cannot query + HSM using an all-caps approach because of 'Other' and 'Unknown' would then never match. + + Because nodes may not have a known architecture, all nodes that are of unknown + architecture count as being of type X86. args: nodes: an iterator of nodes by xnames that correspond to components in HSM. arch: A string representing a corresponding ARCH from HSM. @@ -211,15 +214,17 @@ def _apply_arch(self, nodes, arch): hsm_filter = HSMState() nodes = set(hsm_filter.filter_by_arch(nodes, valid_archs)) if not nodes: - self._log(LOGGER.warning, "After filtering for architecture, no nodes remain to act upon.") + self._log(LOGGER.warning, + "After filtering for architecture, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After filtering for architecture, %d nodes remain to act upon.", len(nodes)) + self._log(LOGGER.debug, + "After filtering for architecture, %d nodes remain to act upon.", len(nodes)) return nodes def _apply_include_disabled(self, nodes): """ - If include_disabled is False for this session, filter out any nodes which are disabled in HSM. - If include_disabled is True, return the node list unchanged. + If include_disabled is False for this session, filter out any nodes which are disabled + in HSM. Otherwise, return the node list unchanged. """ include_disabled = self.session_data.get("include_disabled", False) if include_disabled: @@ -228,9 +233,11 @@ def _apply_include_disabled(self, nodes): hsmfilter = HSMState(enabled=True) nodes = set(hsmfilter._filter(list(nodes))) if not nodes: - self._log(LOGGER.warning, "After removing disabled nodes, no nodes remain to act upon.") + self._log(LOGGER.warning, + "After removing disabled nodes, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After removing disabled nodes, %d nodes remain to act upon.", len(nodes)) + self._log(LOGGER.debug, "After removing disabled nodes, %d nodes remain to act upon.", + len(nodes)) return nodes def _apply_limit(self, nodes): @@ -260,7 +267,8 @@ def _apply_limit(self, nodes): if not nodes: self._log(LOGGER.warning, "After applying limit, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After applying limit, %d nodes remain to act upon.", len(nodes)) + self._log(LOGGER.debug, "After applying limit, %d nodes remain to act upon.", + len(nodes)) return nodes def _apply_tenant_limit(self, nodes): @@ -275,12 +283,14 @@ def _apply_tenant_limit(self, nodes): if not nodes: self._log(LOGGER.warning, "After applying tenant limit, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After applying tenant limit, %d nodes remain to act upon.", len(nodes)) + self._log(LOGGER.debug, "After applying tenant limit, %d nodes remain to act upon.", + len(nodes)) return nodes def _mark_running(self, component_ids): self.bos_client.sessions.update_session( - self.name, self.tenant, {'status': {'status': 'running'}, "components": ",".join(component_ids)}) + self.name, self.tenant, + {'status': {'status': 'running'}, "components": ",".join(component_ids)}) self._log(LOGGER.info, 'Session is running') def _mark_failed(self, err): @@ -325,9 +335,9 @@ def _get_state_from_boot_set(self, boot_set): """ Returns: state: A dictionary containing two keys 'boot_artifacts' and 'configuration'. - 'boot_artifacts' is itself a dictionary containing key/value pairs where the keys are the - boot artifacts (kernel, initrd, rootfs, and boot parameters) and the values are paths to - those artifacts in storage. + 'boot_artifacts' is itself a dictionary containing key/value pairs where the keys are + the boot artifacts (kernel, initrd, rootfs, and boot parameters) and the values are + paths to those artifacts in storage. 'configuration' is a string. """ state = {} @@ -336,7 +346,8 @@ def _get_state_from_boot_set(self, boot_set): artifact_info = image_metadata.artifact_summary boot_artifacts['kernel'] = artifact_info['kernel'] boot_artifacts['initrd'] = image_metadata.initrd.get("link", {}).get("path", "") - boot_artifacts['kernel_parameters'] = self.assemble_kernel_boot_parameters(boot_set, artifact_info) + boot_artifacts['kernel_parameters'] = self.assemble_kernel_boot_parameters(boot_set, + artifact_info) state['boot_artifacts'] = boot_artifacts state['configuration'] = self._get_configuration_from_boot_set(boot_set) return state @@ -376,7 +387,8 @@ def assemble_kernel_boot_parameters(self, boot_set, artifact_info): Inputs: boot_set: A boot set from the session template data artifact_info: The artifact summary from the boot_set. - This is a dictionary containing keys which are boot artifacts (kernel, initrd, roots, and kernel boot parameters). + This is a dictionary containing keys which are boot artifacts (kernel, + initrd, roots, and kernel boot parameters). The values are the paths to those boot artifacts in S3. It also contains the etags for the rootfs and kerenl boot parameters. Returns: @@ -401,14 +413,15 @@ def assemble_kernel_boot_parameters(self, boot_set, artifact_info): artifact_info['boot_parameters_etag']) image_kernel_parameters_object = s3_obj.object - image_kernel_parameters_raw = image_kernel_parameters_object['Body'].read().decode('utf-8') - image_kernel_parameters = image_kernel_parameters_raw.split() + parameters_raw = image_kernel_parameters_object['Body'].read().decode('utf-8') + image_kernel_parameters = parameters_raw.split() if image_kernel_parameters: boot_param_pieces.extend(image_kernel_parameters) except (ClientError, UnicodeDecodeError, S3ObjectNotFound) as error: - self._log(LOGGER.error, "Unable to read file {}. Thus, no kernel boot parameters obtained " - "from image".format(artifact_info['boot_parameters'])) - LOGGER.error(exc_type_msg(error)) + self._log(LOGGER.error, + "Error reading file %s; no kernel boot parameters obtained from image", + artifact_info['boot_parameters']) + self._log(LOGGER.error, exc_type_msg(error)) raise # Parameters from the BOS Session template if the parameters exist. diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index e9f70589..529be818 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -49,8 +49,10 @@ def __init__(self): self.desired_configuration_is_none = DesiredConfigurationIsNone()._match self.desired_configuration_set_in_cfs = DesiredConfigurationSetInCFS()._match self.last_action_is_power_on = LastActionIs(Action.power_on)._match - self.boot_wait_time_elapsed = TimeSinceLastAction(seconds=options.max_boot_wait_time)._match - self.power_on_wait_time_elapsed = TimeSinceLastAction(seconds=options.max_power_on_wait_time)._match + self.boot_wait_time_elapsed = TimeSinceLastAction( + seconds=options.max_boot_wait_time)._match + self.power_on_wait_time_elapsed = TimeSinceLastAction( + seconds=options.max_power_on_wait_time)._match @property def name(self): @@ -77,8 +79,10 @@ def _run(self) -> None: cfs_states = self._get_cfs_components() updated_components = [] # Recreate these filters to pull in the latest options values - self.boot_wait_time_elapsed = TimeSinceLastAction(seconds=options.max_boot_wait_time)._match - self.power_on_wait_time_elapsed = TimeSinceLastAction(seconds=options.max_power_on_wait_time)._match + self.boot_wait_time_elapsed = TimeSinceLastAction( + seconds=options.max_boot_wait_time)._match + self.power_on_wait_time_elapsed = TimeSinceLastAction( + seconds=options.max_power_on_wait_time)._match for component in components: updated_component = self._check_status( component, power_states.get(component['id']), cfs_states.get(component['id'])) @@ -111,7 +115,9 @@ def _check_status(self, component, power_state, cfs_component): state. If its status differs from the status in the database, return this information. """ if power_state and cfs_component: - phase, override, disable, error, action_failed = self._calculate_status(component, power_state, cfs_component) + phase, override, disable, error, action_failed = self._calculate_status(component, + power_state, + cfs_component) else: # If the component cannot be found in pcs or cfs phase = Phase.none @@ -154,7 +160,8 @@ def _check_status(self, component, power_state, cfs_component): if error and error != component.get('error', ''): updated_component['error'] = error update = True - if action_failed and action_failed != component.get('last_action', {}).get('failed', False): + if action_failed and action_failed != component.get('last_action', {}).get('failed', + False): updated_component['last_action'] = {} updated_component['last_action']['failed'] = True update = True @@ -187,7 +194,8 @@ def _calculate_status(self, component, power_state, cfs_component): phase = Phase.none disable = True # Successful state - desired and actual state are off else: - if self.last_action_is_power_on(component) and self.power_on_wait_time_elapsed(component): + if self.last_action_is_power_on(component) and self.power_on_wait_time_elapsed( + component): action_failed = True phase = Phase.powering_on else: @@ -197,27 +205,34 @@ def _calculate_status(self, component, power_state, cfs_component): if not self.desired_configuration_set_in_cfs(component, cfs_component): phase = Phase.configuring elif self.desired_configuration_is_none(component): + # Successful state - booted with the correct artifacts, + # no configuration necessary phase = Phase.none - disable = True # Successful state - booted with the correct artifacts, no configuration necessary + disable = True else: cfs_status = cfs_component.get('configuration_status', '').lower() if cfs_status == 'configured': + # Successful state - booted with the correct artifacts and configured phase = Phase.none - disable = True # Successful state - booted with the correct artifacts and configured + disable = True elif cfs_status == 'failed': + # Failed state - configuration failed phase = Phase.configuring - disable = True # Failed state - configuration failed + disable = True override = Status.failed error = 'cfs configuration failed' elif cfs_status == 'pending': phase = Phase.configuring else: + # Failed state - configuration is no longer set phase = Phase.configuring - disable = True # Failed state - configuration is no longer set + disable = True override = Status.failed - error = f'cfs is not reporting a valid configuration status for this component: {cfs_status}' + error = ('cfs is not reporting a valid configuration status for ' + f'this component: {cfs_status}') else: - if self.last_action_is_power_on(component) and not self.boot_wait_time_elapsed(component): + if self.last_action_is_power_on(component) and not self.boot_wait_time_elapsed( + component): phase = Phase.powering_on else: # Includes both power-off for restarts and ready-recovery scenario diff --git a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py index 2b144baf..a0ad8f28 100644 --- a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py +++ b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py @@ -27,7 +27,8 @@ from bos.common.utils import exc_type_msg from bos.operators.utils.boot_image_metadata import BootImageMetaData, BootImageMetaDataBadRead -from bos.operators.utils.clients.s3 import S3BootArtifacts, S3MissingConfiguration, ArtifactNotFound +from bos.operators.utils.clients.s3 import S3BootArtifacts, S3MissingConfiguration, \ + ArtifactNotFound LOGGER = logging.getLogger('bos.operators.utils.boot_image_metadata.s3_boot_image_metadata') @@ -80,7 +81,8 @@ def metadata(self): try: return self.boot_artifacts.manifest_json except (ClientError, S3MissingConfiguration) as error: - LOGGER.error("Unable to read %s -- Error: %s", self._boot_set.get('path', ''), exc_type_msg(error)) + LOGGER.error("Unable to read %s -- Error: %s", self._boot_set.get('path', ''), + exc_type_msg(error)) raise BootImageMetaDataBadRead(error) from error @property diff --git a/src/bos/operators/utils/clients/bos/base.py b/src/bos/operators/utils/clients/bos/base.py index 5acb769f..abc32a02 100644 --- a/src/bos/operators/utils/clients/bos/base.py +++ b/src/bos/operators/utils/clients/bos/base.py @@ -167,7 +167,8 @@ def update_item(self, item_id, tenant, data): def update_items(self, tenant, data): """Update information for multiple BOS items""" LOGGER.debug("PATCH %s for tenant=%s with body=%s", self.base_url, tenant, data) - response = self.session.patch(self.base_url, json=data, headers=get_new_tenant_header(tenant)) + response = self.session.patch(self.base_url, json=data, + headers=get_new_tenant_header(tenant)) response.raise_for_status() items = json.loads(response.text) return items diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/operators/utils/clients/bos/options.py index e2b10d9a..2313eab4 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/operators/utils/clients/bos/options.py @@ -100,7 +100,8 @@ def disable_components_on_completion(self): @property def cleanup_completed_session_ttl(self): - return self.get_option('cleanup_completed_session_ttl', str, '7d') # Defaults to 7 days (168 hours). + # Defaults to 7 days (168 hours). + return self.get_option('cleanup_completed_session_ttl', str, '7d') @property def clear_stage(self): diff --git a/src/bos/operators/utils/clients/pcs.py b/src/bos/operators/utils/clients/pcs.py index 75f9b4fc..7bc9e6ae 100644 --- a/src/bos/operators/utils/clients/pcs.py +++ b/src/bos/operators/utils/clients/pcs.py @@ -108,7 +108,9 @@ def _power_status(xname=None, power_state_filter=None, management_state_filter=N try: response.raise_for_status() if not response.ok: - raise PowerControlException("Non-2XX response to power_status query: response %s: %s" %(response.status_c)) + raise PowerControlException(f"Non-2XX response ({response.status_code}) to " + f"power_status query; {response.reason} " + f"{compact_response_text(response.text)}") except requests.exceptions.HTTPError as err: raise PowerControlException(err) from err try: @@ -146,7 +148,8 @@ def status(nodes, session=None, **kwargs): power_status_all = _power_status(xname=list(nodes), session=session, **kwargs) for power_status_entry in power_status_all['status']: # If the returned xname has an error, it itself is the status regardless of - # what the powerState field suggests. This is a major departure from how CAPMC handled errors. + # what the powerState field suggests. This is a major departure from how CAPMC + # handled errors. xname = power_status_entry.get('xname', '') if power_status_entry['error']: status_bucket[power_status_entry['error']].add(xname) @@ -159,7 +162,8 @@ def status(nodes, session=None, **kwargs): def node_to_powerstate(nodes, session=None, **kwargs): """ - For an iterable of nodes ; return a dictionary that maps to the current power state for the node in question. + For an iterable of nodes ; return a dictionary that maps to the current power state for + the node in question. """ power_states = {} if not nodes: @@ -176,27 +180,30 @@ def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key """ Interact with PCS to create a request to transition one or more xnames. The transition operation indicates what the desired operation should be, which is a string value containing - one or more of the supported transition names for the given hardware, e.g. 'on', 'off', or 'force-off'. + one or more of the supported transition names for the given hardware, e.g. 'on', 'off', or + 'force-off'. - Once created, one of two responses are returned. A 2XX response results in a transition_start_output - object, or, an invalid request results in a 4XX and subsequent raised PCS exception. + Once created, one of two responses are returned. A 2XX response results in a + transition_start_output object, or, an invalid request results in a 4XX and subsequent + raised PCS exception. Args: xnames: an iterable of xnames operation: A string/enum for what the nodes should transition to - task_deadline_minutes: How long should PCS operate on the nodes to bring them to complete the operation; - typecast to an integer value. - deputy_key: An optional string value that can be used to further handle instructing PCS to perform - state transitions on behalf of a known existing reservation. + task_deadline_minutes: How long should PCS operate on the nodes to bring them to complete + the operation; typecast to an integer value. + deputy_key: An optional string value that can be used to further handle instructing PCS + to perform state transitions on behalf of a known existing reservation. session: An already existing session to use with PCS, if any Returns: - A transition_start_output object, which is a record for the transition that was created. The most important - field of which is the 'transitionID' value, which allows subsequent follow-on to the created request. + A transition_start_output object, which is a record for the transition that was created. + The most important field of which is the 'transitionID' value, which allows subsequent + follow-on to the created request. Raises: - PowerControlException: Any non-nominal response from PCS, typically as a result of an unexpected payload - response, or a failure to create a transition record. + PowerControlException: Any non-nominal response from PCS, typically as a result of an + unexpected payload response, or a failure to create a transition record. PowerControlComponentsEmptyException: No xnames specified """ if not xnames: @@ -204,9 +211,11 @@ def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key "_transition_create called with no xnames! (operation=%s)" % operation) session = session or requests_retry_session() try: - assert operation in set(['On', 'Off', 'Soft-Off', 'Soft-Restart', 'Hard-Restart', 'Init', 'Force-Off']) + assert operation in {'On', 'Off', 'Soft-Off', 'Soft-Restart', 'Hard-Restart', 'Init', + 'Force-Off'} except AssertionError as err: - raise PowerControlSyntaxException("Operation '%s' is not supported or implemented." %(operation)) from err + raise PowerControlSyntaxException( + f"Operation '{operation}' is not supported or implemented.") from err params = {'location': [], 'operation': operation} if task_deadline_minutes: params['taskDeadlineMinutes'] = int(task_deadline_minutes) @@ -222,7 +231,11 @@ def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key try: response.raise_for_status() if not response.ok: - raise PowerControlException("Non-2XX response to power_status query: response %s: %s" %(response.status_c)) + raise PowerControlException(f"Non-2XX response ({response.status_code}) to " + f"{operation} power transition creation; " + f"{response.reason} " + f"{compact_response_text(response.text)}") + except requests.exceptions.HTTPError as err: raise PowerControlException(err) from err try: @@ -239,7 +252,8 @@ def power_on(nodes, session=None, task_deadline_minutes=1, **kwargs): if not nodes: raise PowerControlComponentsEmptyException("power_on called with no nodes!") session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='On', task_deadline_minutes=task_deadline_minutes, + return _transition_create(xnames=nodes, operation='On', + task_deadline_minutes=task_deadline_minutes, session=session, **kwargs) def power_off(nodes, session=None, task_deadline_minutes=1, **kwargs): """ @@ -249,7 +263,8 @@ def power_off(nodes, session=None, task_deadline_minutes=1, **kwargs): if not nodes: raise PowerControlComponentsEmptyException("power_off called with no nodes!") session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='Off', task_deadline_minutes=task_deadline_minutes, + return _transition_create(xnames=nodes, operation='Off', + task_deadline_minutes=task_deadline_minutes, session=session, **kwargs) def soft_off(nodes, session=None, task_deadline_minutes=1, **kwargs): """ @@ -259,7 +274,8 @@ def soft_off(nodes, session=None, task_deadline_minutes=1, **kwargs): if not nodes: raise PowerControlComponentsEmptyException("soft_off called with no nodes!") session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='Soft-Off', task_deadline_minutes=task_deadline_minutes, + return _transition_create(xnames=nodes, operation='Soft-Off', + task_deadline_minutes=task_deadline_minutes, session=session, **kwargs) def force_off(nodes, session=None, task_deadline_minutes=1, **kwargs): """ @@ -269,5 +285,6 @@ def force_off(nodes, session=None, task_deadline_minutes=1, **kwargs): if not nodes: raise PowerControlComponentsEmptyException("force_off called with no nodes!") session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='Force-Off', task_deadline_minutes=task_deadline_minutes, + return _transition_create(xnames=nodes, operation='Force-Off', + task_deadline_minutes=task_deadline_minutes, session=session, **kwargs) diff --git a/src/bos/operators/utils/rootfs/factory.py b/src/bos/operators/utils/rootfs/factory.py index 86a3064b..4086922d 100644 --- a/src/bos/operators/utils/rootfs/factory.py +++ b/src/bos/operators/utils/rootfs/factory.py @@ -38,8 +38,10 @@ def __init__(self, boot_set, artifact_info): Inputs: boot_set: A boot set from the session template data artifact_info: The artifact summary from the boot_set. - This is a dictionary containing keys which are boot artifacts (kernel, initrd, roots, and kernel boot parameters) - the values are the paths to those boot artifacts in S3. It also contains the etags for the rootfs and kerenl boot parameters. + This is a dictionary containing keys which are boot artifacts + (kernel, initrd, roots, and kernel boot parameters); + the values are the paths to those boot artifacts in S3. + It also contains the etags for the rootfs and kerenl boot parameters. """ self.boot_set = boot_set self.artifact_info = artifact_info @@ -61,7 +63,8 @@ def __call__(self): module = importlib.import_module(provider_module) except ModuleNotFoundError as mnfe: # This is pretty much unrecoverable at this stage of development; make note and raise - LOGGER.error("Provider provisioning mechanism '%s' not yet implemented or not found.", provider_name) + LOGGER.error("Provider provisioning mechanism '%s' not yet implemented or not found.", + provider_name) raise ProviderNotImplemented(mnfe) from mnfe class_def = getattr(module, provider_classname) diff --git a/src/bos/reporter/client.py b/src/bos/reporter/client.py index 4281953e..f32a8574 100644 --- a/src/bos/reporter/client.py +++ b/src/bos/reporter/client.py @@ -65,7 +65,9 @@ def get_auth_token(path='/opt/cray/auth-utils/bin/get-auth-token'): out = out.rstrip('\n') return out except subprocess.CalledProcessError as e: - LOGGER.error('get_auth_token failed to retrieve authorization token: code=%d: error=%s', e.returncode, e.output) + LOGGER.error( + 'get_auth_token failed to retrieve authorization token: code=%d: error=%s', + e.returncode, e.output) except Exception: LOGGER.exception('Unexpected exception') LOGGER.info("Spire Token not yet available; retrying in a few seconds.") diff --git a/src/bos/reporter/components/state.py b/src/bos/reporter/components/state.py index eabc5c3a..aa5eaf13 100644 --- a/src/bos/reporter/components/state.py +++ b/src/bos/reporter/components/state.py @@ -72,8 +72,10 @@ def patch_component(component, properties, session=None): json_response = json.loads(response.text) raise UnknownComponent(json_response['detail']) from hpe except json.JSONDecodeError as jde: - raise UnrecognizedResponse("BOS returned a non-json response: %s\n%s" % (response.text, jde)) from jde - LOGGER.warning("Unexpected response from '%s':\n%s: %s", component_endpoint, response.status_code, response.text) + raise UnrecognizedResponse( + f"BOS returned a non-json response: {response.text}\n{jde}") from jde + LOGGER.warning("Unexpected response from '%s':\n%s: %s", component_endpoint, + response.status_code, response.text) raise BOSComponentException(hpe) from hpe diff --git a/src/bos/reporter/node_identity.py b/src/bos/reporter/node_identity.py index 85934e98..88a8dd85 100644 --- a/src/bos/reporter/node_identity.py +++ b/src/bos/reporter/node_identity.py @@ -61,7 +61,8 @@ def identity_from_environment(): try: return os.environ[ident_string] except KeyError as exc: - raise UnknownIdentity("Node identity not passed in via environment '%s'" % (ident_string)) from exc + raise UnknownIdentity( + f"Node identity not passed in via environment '{ident_string}'") from exc def read_identity(): diff --git a/src/bos/server/controllers/v2/boot_set.py b/src/bos/server/controllers/v2/boot_set.py index f0f8ce36..f45fd081 100644 --- a/src/bos/server/controllers/v2/boot_set.py +++ b/src/bos/server/controllers/v2/boot_set.py @@ -106,7 +106,8 @@ def validate_boot_sets(session_template: dict, try: artifact = getattr(image_metadata.boot_artifacts, boot_artifact) if not artifact: - raise ArtifactNotFound(f"Session template: '{template_name}' boot set: '{bs_name}' " \ + raise ArtifactNotFound(f"Session template: '{template_name}' " + f"boot set: '{bs_name}' " f"does not contain a {boot_artifact}.") path = artifact ['link']['path'] etag = artifact['link']['etag'] diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 2c848f03..674384ce 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -26,14 +26,16 @@ import connexion from bos.common.utils import exc_type_msg, get_current_timestamp -from bos.common.tenant_utils import get_tenant_from_header, get_tenant_component_set, tenant_error_handler, get_tenant_aware_key +from bos.common.tenant_utils import get_tenant_from_header, get_tenant_component_set, \ + tenant_error_handler, get_tenant_aware_key from bos.common.values import Phase, Action, Status, EMPTY_STAGED_STATE, EMPTY_BOOT_ARTIFACTS from bos.server import redis_db_utils as dbutils from bos.server.controllers.v2.options import get_v2_options_data from bos.server.dbs.boot_artifacts import get_boot_artifacts, BssTokenUnknown -from bos.server.models.v2_component import V2Component as Component # noqa: E501 -from bos.server.models.v2_component_array import V2ComponentArray as ComponentArray # noqa: E501 -from bos.server.models.v2_components_update import V2ComponentsUpdate as ComponentsUpdate # noqa: E501 +from bos.server.models.v2_component import V2Component as Component # noqa: E501 +from bos.server.models.v2_component_array import V2ComponentArray as ComponentArray # noqa: E501 +from bos.server.models.v2_components_update import V2ComponentsUpdate as \ + ComponentsUpdate # noqa: E501 LOGGER = logging.getLogger('bos.server.controllers.v2.components') DB = dbutils.get_wrapper(db='components') @@ -42,7 +44,8 @@ @tenant_error_handler @dbutils.redis_error_handler -def get_v2_components(ids="", enabled=None, session=None, staged_session=None, phase=None, status=None): +def get_v2_components(ids="", enabled=None, session=None, staged_session=None, phase=None, + status=None): """Used by the GET /components API operation Allows filtering using a comma separated list of ids. @@ -61,9 +64,11 @@ def get_v2_components(ids="", enabled=None, session=None, staged_session=None, p detail=str(err)) tenant = get_tenant_from_header() LOGGER.debug("GET /v2/components for tenant=%s with %d IDs specified", tenant, len(id_list)) - response = get_v2_components_data(id_list=id_list, enabled=enabled, session=session, staged_session=staged_session, + response = get_v2_components_data(id_list=id_list, enabled=enabled, session=session, + staged_session=staged_session, phase=phase, status=status, tenant=tenant) - LOGGER.debug("GET /v2/components returning data for tenant=%s on %d components", tenant, len(response)) + LOGGER.debug("GET /v2/components returning data for tenant=%s on %d components", tenant, + len(response)) for component in response: del_timestamp(component) return response, 200 @@ -85,13 +90,17 @@ def get_v2_components_data(id_list=None, enabled=None, session=None, staged_sess # TODO: On large scale systems, this response may be too large # and require paging to be implemented response = DB.get_all() - # The status must be set before using _matches_filter as the status is one of the matching criteria. + # The status must be set before using _matches_filter as the status is one of the + # matching criteria. response = [_set_status(r) for r in response if r] - if enabled is not None or session is not None or staged_session is not None or phase is not None or status is not None: - response = [r for r in response if _matches_filter(r, enabled, session, staged_session, phase, status)] + if enabled is not None or session is not None or staged_session is not None or \ + phase is not None or status is not None: + response = [r for r in response if _matches_filter(r, enabled, session, staged_session, + phase, status)] if tenant: tenant_components = get_tenant_component_set(tenant) - limited_response = [component for component in response if component["id"] in tenant_components] + limited_response = [component for component in response + if component["id"] in tenant_components] response = limited_response return response @@ -111,7 +120,9 @@ def _calculate_status(data): Calculates and returns the status of a component """ if not 'status' in data: - LOGGER.debug("No status in data: %s. This will have the effect of clearing any pre-existing phase.", data) + LOGGER.debug( + "No status in data: %s. This will have the effect of clearing any pre-existing phase.", + data) status_data = data.get('status', {}) override = status_data.get('status_override', '') if override: @@ -137,7 +148,8 @@ def _calculate_status(data): elif phase == Phase.configuring: status = Status.configuring - LOGGER.debug("Component: %s Last action: %s Phase: %s Status: %s", component, last_action, phase, status) + LOGGER.debug("Component: %s Last action: %s Phase: %s Status: %s", component, last_action, + phase, status) return status @@ -146,7 +158,8 @@ def _matches_filter(data, enabled, session, staged_session, phase, status): return False if session is not None and data.get('session', None) != session: return False - if staged_session is not None and data.get('staged_state', {}).get('session', None) != staged_session: + if staged_session is not None and \ + data.get('staged_state', {}).get('session', None) != staged_session: return False status_data = data.get('status') if phase is not None and status_data.get('phase') != phase: @@ -288,7 +301,9 @@ def patch_v2_components_dict(data): status=404, title="Component not found.", detail="Component {} could not be found".format(component_id)) elif session: - id_list = [component["id"] for component in get_v2_components_data(session=session, tenant=get_tenant_from_header())] + id_list = [component["id"] for component in get_v2_components_data( + session=session, + tenant=get_tenant_from_header())] LOGGER.debug("patch_v2_components_dict: %d IDs found for specified session", len(id_list)) else: LOGGER.warning("No filter provided") @@ -441,7 +456,8 @@ def post_v2_apply_staged(): else: response["ignored"].append(xname) except Exception: - LOGGER.exception("An error was encountered while attempting to apply stage for node %s", xname) + LOGGER.exception( + "An error was encountered while attempting to apply stage for node %s", xname) response["failed"].append(xname) except Exception as err: LOGGER.error("Error parsing request data: %s", exc_type_msg(err)) @@ -511,11 +527,13 @@ def _set_state_from_staged(data): _copy_staged_to_desired(data) elif operation == "boot": if not all(staged_state.get("boot_artifacts", {}).values()): - raise Exception("Staged operation is boot but some boot artifacts have not been specified") + raise Exception( + "Staged operation is boot but some boot artifacts have not been specified") _copy_staged_to_desired(data) elif operation == "reboot": if not all(staged_state.get("boot_artifacts", {}).values()): - raise Exception("Staged operation is reboot but some boot artifacts have not been specified") + raise Exception( + "Staged operation is reboot but some boot artifacts have not been specified") _copy_staged_to_desired(data) data["actual_state"] = { "boot_artifacts": EMPTY_BOOT_ARTIFACTS, @@ -588,7 +606,8 @@ def del_timestamp(data: dict): def _set_last_updated(data): timestamp = get_current_timestamp() for section in ['actual_state', 'desired_state', 'staged_state', 'last_action']: - if section in data and isinstance(data[section], dict) and data[section].keys() != {"bss_token"}: + if section in data and isinstance(data[section], + dict) and data[section].keys() != {"bss_token"}: data[section]['last_updated'] = timestamp return data diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index f2356b3c..217a5f2a 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -30,7 +30,8 @@ import connexion from connexion.lifecycle import ConnexionResponse -from bos.common.tenant_utils import get_tenant_from_header, get_tenant_aware_key, reject_invalid_tenant +from bos.common.tenant_utils import get_tenant_from_header, get_tenant_aware_key, \ + reject_invalid_tenant from bos.common.utils import exc_type_msg, get_current_time, get_current_timestamp, load_timestamp from bos.common.values import Phase, Status from bos.server import redis_db_utils as dbutils @@ -211,8 +212,9 @@ def delete_v2_session(session_id): # noqa: E501 @dbutils.redis_error_handler def delete_v2_sessions(min_age=None, max_age=None, status=None): # noqa: E501 - LOGGER.debug("DELETE /v2/sessions invoked delete_v2_sessions with min_age=%s max_age=%s status=%s", - min_age, max_age, status) + LOGGER.debug( + "DELETE /v2/sessions invoked delete_v2_sessions with min_age=%s max_age=%s status=%s", + min_age, max_age, status) tenant = get_tenant_from_header() try: sessions = _get_filtered_sessions(tenant=tenant, min_age=min_age, max_age=max_age, @@ -252,7 +254,8 @@ def get_v2_session_status(session_id): # noqa: E501 detail="Session {} could not be found".format(session_id)) session = DB.get(session_key) if session.get("status", {}).get("status") == "complete" and session_key in STATUS_DB: - # If the session is complete and the status is saved, return the status from completion time + # If the session is complete and the status is saved, + # return the status from completion time return STATUS_DB.get(session_key), 200 return _get_v2_session_status(session_key, session), 200 @@ -293,7 +296,8 @@ def _get_filtered_sessions(tenant, min_age, max_age, status): LOGGER.warning('Unable to parse max_age: %s', max_age) raise ParsingException(e) from e if any([min_start, max_start, status, tenant]): - response = [r for r in response if _matches_filter(r, tenant, min_start, max_start, status)] + response = [r for r in response if _matches_filter(r, tenant, min_start, max_start, + status)] return response @@ -323,11 +327,18 @@ def _get_v2_session_status(session_key, session=None): staged_components = get_v2_components_data(staged_session=session_id, tenant=tenant_id) num_managed_components = len(components) + len(staged_components) if num_managed_components: - component_phase_counts = Counter([c.get('status', {}).get('phase') for c in components if (c.get('enabled') and c.get('status').get('status_override') != Status.on_hold)]) - component_phase_counts['successful'] = len([c for c in components if c.get('status', {}).get('status') == Status.stable]) - component_phase_counts['failed'] = len([c for c in components if c.get('status', {}).get('status') == Status.failed]) + component_phase_counts = Counter([ + c.get('status', {}).get('phase') for c in components + if (c.get('enabled') and + c.get('status').get('status_override') != Status.on_hold)]) + component_phase_counts['successful'] = len([ + c for c in components if c.get('status',{}).get('status') == Status.stable]) + component_phase_counts['failed'] = len([ + c for c in components if c.get('status', {}).get('status') == Status.failed]) component_phase_counts['staged'] = len(staged_components) - component_phase_percents = {phase: (component_phase_counts[phase] / num_managed_components) * 100 for phase in component_phase_counts} + component_phase_percents = { + phase: (component_phase_counts[phase] / num_managed_components) * 100 + for phase in component_phase_counts} else: component_phase_percents = {} component_errors_data = defaultdict(set) @@ -351,7 +362,9 @@ def _get_v2_session_status(session_key, session=None): 'status': session_status.get('status', ''), 'managed_components_count': num_managed_components, 'phases': { - 'percent_complete': round(component_phase_percents.get('successful', 0) + component_phase_percents.get('failed', 0), 2), + 'percent_complete': round( + component_phase_percents.get('successful', + 0) + component_phase_percents.get('failed', 0), 2), 'percent_powering_on': round(component_phase_percents.get(Phase.powering_on, 0), 2), 'percent_powering_off': round(component_phase_percents.get(Phase.powering_off, 0), 2), 'percent_configuring': round(component_phase_percents.get(Phase.configuring, 0), 2), diff --git a/src/bos/server/controllers/v2/sessiontemplates.py b/src/bos/server/controllers/v2/sessiontemplates.py index 25955ea4..0c3aab70 100644 --- a/src/bos/server/controllers/v2/sessiontemplates.py +++ b/src/bos/server/controllers/v2/sessiontemplates.py @@ -24,9 +24,10 @@ import logging import connexion -from bos.common.tenant_utils import get_tenant_from_header, get_tenant_aware_key, reject_invalid_tenant +from bos.common.tenant_utils import get_tenant_from_header, get_tenant_aware_key, \ + reject_invalid_tenant from bos.common.utils import exc_type_msg -from bos.server.models.v2_session_template import V2SessionTemplate as SessionTemplate # noqa: E501 +from bos.server.models.v2_session_template import V2SessionTemplate as SessionTemplate # noqa: E501 from bos.server import redis_db_utils as dbutils from bos.server.utils import _canonize_xname from .boot_set import validate_boot_sets @@ -170,7 +171,8 @@ def delete_v2_sessiontemplate(session_template_id): Delete the session template by session template ID """ - LOGGER.debug("DELETE /v2/sessiontemplates/%s invoked delete_v2_sessiontemplate", session_template_id) + LOGGER.debug("DELETE /v2/sessiontemplates/%s invoked delete_v2_sessiontemplate", + session_template_id) template_key = get_tenant_aware_key(session_template_id, get_tenant_from_header()) if template_key not in DB: LOGGER.warning("Session template not found: %s", session_template_id) @@ -187,7 +189,8 @@ def patch_v2_sessiontemplate(session_template_id): Patch the session template by session template ID """ - LOGGER.debug("PATCH /v2/sessiontemplates/%s invoked patch_v2_sessiontemplate", session_template_id) + LOGGER.debug("PATCH /v2/sessiontemplates/%s invoked patch_v2_sessiontemplate", + session_template_id) template_key = get_tenant_aware_key(session_template_id, get_tenant_from_header()) if template_key not in DB: LOGGER.warning("Session template not found: %s", session_template_id) @@ -238,7 +241,8 @@ def validate_v2_sessiontemplate(session_template_id: str): Validate a V2 session template. Look for missing elements or errors that would prevent a session from being launched using this template. """ - LOGGER.debug("GET /v2/sessiontemplatesvalid/%s invoked validate_v2_sessiontemplate", session_template_id) + LOGGER.debug("GET /v2/sessiontemplatesvalid/%s invoked validate_v2_sessiontemplate", + session_template_id) data, status_code = get_v2_sessiontemplate(session_template_id) if status_code != 200: diff --git a/src/bos/server/redis_db_utils.py b/src/bos/server/redis_db_utils.py index 4470fb7a..a8666920 100644 --- a/src/bos/server/redis_db_utils.py +++ b/src/bos/server/redis_db_utils.py @@ -31,7 +31,8 @@ from bos.common.utils import exc_type_msg LOGGER = logging.getLogger(__name__) -DATABASES = ["options", "components", "session_templates", "sessions", "bss_tokens_boot_artifacts", "session_status"] # Index is the db id. +DATABASES = ["options", "components", "session_templates", "sessions", "bss_tokens_boot_artifacts", + "session_status"] # Index is the db id. DB_HOST = 'cray-bos-db' DB_PORT = 6379 From 7a5437f64e87c55a9988453d4380640de94b08b0 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 14:03:02 -0400 Subject: [PATCH 19/29] Linting: line-too-long --- src/bos/operators/utils/clients/pcs.py | 3 ++- src/bos/server/controllers/v2/sessions.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/bos/operators/utils/clients/pcs.py b/src/bos/operators/utils/clients/pcs.py index 7bc9e6ae..36231d77 100644 --- a/src/bos/operators/utils/clients/pcs.py +++ b/src/bos/operators/utils/clients/pcs.py @@ -176,7 +176,8 @@ def node_to_powerstate(nodes, session=None, **kwargs): power_states[node] = pstatus return power_states -def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key=None, session=None): +def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key=None, + session=None): """ Interact with PCS to create a request to transition one or more xnames. The transition operation indicates what the desired operation should be, which is a string value containing diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index 217a5f2a..c0b55fbd 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -328,9 +328,10 @@ def _get_v2_session_status(session_key, session=None): num_managed_components = len(components) + len(staged_components) if num_managed_components: component_phase_counts = Counter([ - c.get('status', {}).get('phase') for c in components - if (c.get('enabled') and - c.get('status').get('status_override') != Status.on_hold)]) + c.get('status', {}).get('phase') + for c in components + if (c.get('enabled') and + c.get('status').get('status_override') != Status.on_hold)]) component_phase_counts['successful'] = len([ c for c in components if c.get('status',{}).get('status') == Status.stable]) component_phase_counts['failed'] = len([ From 0e8d3d96029cbb403422a4ff770dfd38d94f7c35 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 14:33:50 -0400 Subject: [PATCH 20/29] Linting: consider-using-f-string --- src/bos/common/tenant_utils.py | 4 ++-- src/bos/common/utils.py | 2 +- src/bos/operators/session_setup.py | 6 +++--- src/bos/operators/utils/clients/bos/base.py | 4 ++-- src/bos/operators/utils/clients/bos/options.py | 5 +++-- .../utils/clients/bos/sessions_status.py | 2 +- src/bos/operators/utils/clients/bss.py | 4 ++-- src/bos/operators/utils/clients/cfs.py | 4 ++-- src/bos/operators/utils/clients/hsm.py | 4 ++-- src/bos/operators/utils/clients/pcs.py | 8 ++++---- src/bos/operators/utils/clients/s3.py | 2 +- src/bos/operators/utils/rootfs/__init__.py | 2 +- src/bos/operators/utils/rootfs/baserootfs.py | 6 +++--- src/bos/operators/utils/rootfs/factory.py | 4 ++-- src/bos/reporter/__init__.py | 5 ++--- src/bos/reporter/client.py | 3 ++- src/bos/reporter/components/__init__.py | 3 ++- src/bos/reporter/components/state.py | 2 +- src/bos/reporter/proc_cmdline.py | 2 +- src/bos/server/controllers/v2/components.py | 12 ++++++------ src/bos/server/controllers/v2/sessions.py | 16 ++++++++-------- .../server/controllers/v2/sessiontemplates.py | 6 +++--- 22 files changed, 54 insertions(+), 52 deletions(-) diff --git a/src/bos/common/tenant_utils.py b/src/bos/common/tenant_utils.py index db06921a..013246f8 100644 --- a/src/bos/common/tenant_utils.py +++ b/src/bos/common/tenant_utils.py @@ -34,8 +34,8 @@ TENANT_HEADER = "Cray-Tenant-Name" SERVICE_NAME = 'cray-tapms/v1alpha2' -BASE_ENDPOINT = "%s://%s" % (PROTOCOL, SERVICE_NAME) -TENANT_ENDPOINT = "%s/tenants" % BASE_ENDPOINT ## CASMPET-6433 changed this from tenant to tenants +BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}" +TENANT_ENDPOINT = f"{BASE_ENDPOINT}/tenants" # CASMPET-6433 changed this from tenant to tenants class InvalidTenantException(Exception): diff --git a/src/bos/common/utils.py b/src/bos/common/utils.py index c134a528..d1cf03b1 100644 --- a/src/bos/common/utils.py +++ b/src/bos/common/utils.py @@ -95,7 +95,7 @@ def requests_retry_session(retries=10, backoff_factor=0.5, adapter = TimeoutHTTPAdapter(max_retries=retry, timeout=(connect_timeout, read_timeout)) # Must mount to http:// # Mounting to only http will not work! - session.mount("%s://" % protocol, adapter) + session.mount(f"{protocol}://", adapter) return session diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 2eeca39e..91e0b2ad 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -302,10 +302,10 @@ def _mark_failed(self, err): self.name, self.tenant, {'status': {'error': err}}) sco = SessionCompletionOperator() sco._mark_session_complete(self.name, self.tenant) - self._log(LOGGER.info, f'Session {self.name} has failed.') + self._log(LOGGER.info, 'Session %s has failed.', self.name) def _log(self, logger, message, *xargs): - logger('Session {}: {}'.format(self.name, message), *xargs) + logger(f'Session {self.name}: {message}', *xargs) # Operations def _operate(self, component_id, state): @@ -439,7 +439,7 @@ def assemble_kernel_boot_parameters(self, boot_set, artifact_info): boot_param_pieces.append(nmd_parameters) # Add the bos actual state ttl value so nodes know how frequently to report - boot_param_pieces.append('bos_update_frequency=%s' % (options.component_actual_state_ttl)) + boot_param_pieces.append(f'bos_update_frequency={options.component_actual_state_ttl}') return ' '.join(boot_param_pieces) diff --git a/src/bos/operators/utils/clients/bos/base.py b/src/bos/operators/utils/clients/bos/base.py index abc32a02..11d05d29 100644 --- a/src/bos/operators/utils/clients/bos/base.py +++ b/src/bos/operators/utils/clients/bos/base.py @@ -33,7 +33,7 @@ API_VERSION = 'v2' SERVICE_NAME = 'cray-bos' -BASE_ENDPOINT = "%s://%s/%s" % (PROTOCOL, SERVICE_NAME, API_VERSION) +BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{API_VERSION}" def log_call_errors(func): @@ -63,7 +63,7 @@ class BaseBosEndpoint: ENDPOINT = '' def __init__(self): - self.base_url = "%s/%s" % (BASE_ENDPOINT, self.ENDPOINT) + self.base_url = f"{BASE_ENDPOINT}/{self.ENDPOINT}" self.session = requests_retry_session() @log_call_errors diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/operators/utils/clients/bos/options.py index 2313eab4..ed15522d 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/operators/utils/clients/bos/options.py @@ -30,7 +30,8 @@ from bos.operators.utils.clients.bos.base import BASE_ENDPOINT LOGGER = logging.getLogger('bos.operators.utils.clients.bos.options') -ENDPOINT = "%s/%s" % (BASE_ENDPOINT, __name__.lower().rsplit('.', maxsplit=1)[-1]) +__name = __name__.lower().rsplit('.', maxsplit=1)[-1] +ENDPOINT = f"{BASE_ENDPOINT}/{__name}" class Options: @@ -68,7 +69,7 @@ def get_option(self, key, value_type, default): return value_type(self.options[key]) if default: return value_type(default) - raise KeyError('Option {} not found and no default exists'.format(key)) + raise KeyError(f'Option {key} not found and no default exists') @property def logging_level(self): diff --git a/src/bos/operators/utils/clients/bos/sessions_status.py b/src/bos/operators/utils/clients/bos/sessions_status.py index 36a83a91..4fae6b4d 100644 --- a/src/bos/operators/utils/clients/bos/sessions_status.py +++ b/src/bos/operators/utils/clients/bos/sessions_status.py @@ -35,7 +35,7 @@ class SessionStatusEndpoint: ENDPOINT = 'sessions' def __init__(self): - self.base_url = "%s/%s" % (BASE_ENDPOINT, self.ENDPOINT) + self.base_url = f"{BASE_ENDPOINT}/{self.ENDPOINT}" @log_call_errors def get_session_status(self, session_id, tenant): diff --git a/src/bos/operators/utils/clients/bss.py b/src/bos/operators/utils/clients/bss.py index d3e7b13a..0b33c2f2 100644 --- a/src/bos/operators/utils/clients/bss.py +++ b/src/bos/operators/utils/clients/bss.py @@ -30,7 +30,7 @@ LOGGER = logging.getLogger(__name__) SERVICE_NAME = 'cray-bss' -ENDPOINT = "%s://%s/boot/v1" % (PROTOCOL, SERVICE_NAME) +ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/boot/v1" def set_bss(node_set, kernel_params, kernel, initrd, session=None): @@ -67,7 +67,7 @@ def set_bss(node_set, kernel_params, kernel, initrd, session=None): session = session or requests_retry_session() LOGGER.info("Params: %s", kernel_params) - url = "%s/bootparameters" % (ENDPOINT) + url = f"{ENDPOINT}/bootparameters" # Assignment payload payload = {"hosts": list(node_set), diff --git a/src/bos/operators/utils/clients/cfs.py b/src/bos/operators/utils/clients/cfs.py index 44c040bb..6c532b5a 100644 --- a/src/bos/operators/utils/clients/cfs.py +++ b/src/bos/operators/utils/clients/cfs.py @@ -28,8 +28,8 @@ from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL SERVICE_NAME = 'cray-cfs-api' -BASE_ENDPOINT = "%s://%s/v3" % (PROTOCOL, SERVICE_NAME) -COMPONENTS_ENDPOINT = "%s/components" % BASE_ENDPOINT +BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/v3" +COMPONENTS_ENDPOINT = f"{BASE_ENDPOINT}/components" LOGGER = logging.getLogger('bos.operators.utils.clients.cfs') diff --git a/src/bos/operators/utils/clients/hsm.py b/src/bos/operators/utils/clients/hsm.py index 39569e22..b44e4dfe 100644 --- a/src/bos/operators/utils/clients/hsm.py +++ b/src/bos/operators/utils/clients/hsm.py @@ -31,7 +31,7 @@ from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL SERVICE_NAME = 'cray-smd' -BASE_ENDPOINT = "%s://%s/hsm/v2/" % (PROTOCOL, SERVICE_NAME) +BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/hsm/v2/" ENDPOINT = os.path.join(BASE_ENDPOINT, 'State/Components/Query') VERIFY = True @@ -54,7 +54,7 @@ def read_all_node_xnames(): have been discovered; return these as a set. """ session = requests_retry_session() - endpoint = '%s/State/Components/' % (BASE_ENDPOINT) + endpoint = f'{BASE_ENDPOINT}/State/Components/' LOGGER.debug("GET %s", endpoint) try: response = session.get(endpoint) diff --git a/src/bos/operators/utils/clients/pcs.py b/src/bos/operators/utils/clients/pcs.py index 36231d77..fddafdd7 100644 --- a/src/bos/operators/utils/clients/pcs.py +++ b/src/bos/operators/utils/clients/pcs.py @@ -35,9 +35,9 @@ SERVICE_NAME = 'cray-power-control' POWER_CONTROL_VERSION = 'v1' -ENDPOINT = "%s://%s" % (PROTOCOL, SERVICE_NAME) -POWER_STATUS_ENDPOINT = '%s/power-status' % (ENDPOINT) -TRANSITION_ENDPOINT = "%s/transitions" %(ENDPOINT) +ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}" +POWER_STATUS_ENDPOINT = f'{ENDPOINT}/power-status' +TRANSITION_ENDPOINT = f"{ENDPOINT}/transitions" LOGGER = logging.getLogger('bos.operators.utils.clients.pcs') @@ -209,7 +209,7 @@ def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key """ if not xnames: raise PowerControlComponentsEmptyException( - "_transition_create called with no xnames! (operation=%s)" % operation) + f"_transition_create called with no xnames! (operation={operation})") session = session or requests_retry_session() try: assert operation in {'On', 'Off', 'Soft-Off', 'Soft-Restart', 'Hard-Restart', 'Init', diff --git a/src/bos/operators/utils/clients/s3.py b/src/bos/operators/utils/clients/s3.py index c093f27e..4b9afb3b 100644 --- a/src/bos/operators/utils/clients/s3.py +++ b/src/bos/operators/utils/clients/s3.py @@ -283,7 +283,7 @@ def _get_artifact(self, artifact_type): LOGGER.info(msg) raise ArtifactNotFound(msg) if len(artifacts) > 1: - msg = "Multiple %s artifacts found in the manifest." % artifact_type + msg = f"Multiple {artifact_type} artifacts found in the manifest." LOGGER.info(msg) raise TooManyArtifacts(msg) return artifacts[0] diff --git a/src/bos/operators/utils/rootfs/__init__.py b/src/bos/operators/utils/rootfs/__init__.py index 7a360e75..5af6171a 100644 --- a/src/bos/operators/utils/rootfs/__init__.py +++ b/src/bos/operators/utils/rootfs/__init__.py @@ -76,7 +76,7 @@ def __str__(self): fields.append(rootfs_provider_passthrough) stripped_fields = [field for field in fields if field] - return "root={}".format(self.DELIMITER.join(fields)) if stripped_fields else '' + return f"root={self.DELIMITER.join(fields)}" if stripped_fields else '' @property def provider_field(self): diff --git a/src/bos/operators/utils/rootfs/baserootfs.py b/src/bos/operators/utils/rootfs/baserootfs.py index 89152c48..84ff23b7 100644 --- a/src/bos/operators/utils/rootfs/baserootfs.py +++ b/src/bos/operators/utils/rootfs/baserootfs.py @@ -48,7 +48,7 @@ def nmd_field(self): """ fields = [] if self.provider_field: - fields.append("url=%s" % self.provider_field) + fields.append(f"url={self.provider_field}") if self.provider_field_id: - fields.append("etag=%s" % self.provider_field_id) - return "nmd_data={}".format(",".join(fields)) if fields else '' + fields.append(f"etag={self.provider_field_id}") + return f"nmd_data={','.join(fields)}" if fields else '' diff --git a/src/bos/operators/utils/rootfs/factory.py b/src/bos/operators/utils/rootfs/factory.py index 4086922d..a788ee61 100644 --- a/src/bos/operators/utils/rootfs/factory.py +++ b/src/bos/operators/utils/rootfs/factory.py @@ -51,8 +51,8 @@ def __call__(self): if provider_name: # When a provisioning protocol is specified... - provider_module = 'bos.operators.utils.rootfs.{}'.format(provider_name) - provider_classname = '{}Provider'.format(provider_name.upper()) + provider_module = f'bos.operators.utils.rootfs.{provider_name}' + provider_classname = f'{provider_name.upper()}Provider' else: # none specified or blank provider_module = 'bos.operators.utils.rootfs' diff --git a/src/bos/reporter/__init__.py b/src/bos/reporter/__init__.py index 13a05652..ad2f28a1 100644 --- a/src/bos/reporter/__init__.py +++ b/src/bos/reporter/__init__.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2020-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2020-2022, 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -28,8 +28,7 @@ API_GATEWAY_DNS_NAME = os.environ.get('API_GATEWAY_HOST', 'api-gw-service-nmn.local') SERVICE = 'bos' API_VERSION = 'v2' -ENDPOINT = '%s://%s/apis/%s/%s' % (PROTOCOL, API_GATEWAY_DNS_NAME, - SERVICE, API_VERSION) +ENDPOINT = f'{PROTOCOL}://{API_GATEWAY_DNS_NAME}/apis/{SERVICE}/{API_VERSION}' class BOSException(Exception): diff --git a/src/bos/reporter/client.py b/src/bos/reporter/client.py index f32a8574..a5facc10 100644 --- a/src/bos/reporter/client.py +++ b/src/bos/reporter/client.py @@ -90,5 +90,6 @@ def requests_retry_session(retries=10, connect=10, backoff_factor=0.5, ) adapter = HTTPAdapter(max_retries=retry) session.mount(PROTOCOL, adapter) - session.headers.update({'Authorization': 'Bearer %s' % (get_auth_token())}) + auth_token = get_auth_token() + session.headers.update({'Authorization': f'Bearer {auth_token}'}) return session diff --git a/src/bos/reporter/components/__init__.py b/src/bos/reporter/components/__init__.py index cbc473b2..50829a8d 100644 --- a/src/bos/reporter/components/__init__.py +++ b/src/bos/reporter/components/__init__.py @@ -24,7 +24,8 @@ from bos.reporter import BOSException from bos.reporter import ENDPOINT as BOS_ENDPOINT -ENDPOINT = "%s/%s" % (BOS_ENDPOINT, __name__.rsplit('.', maxsplit=1)[-1]) +__name = __name__.rsplit('.', maxsplit=1)[-1] +ENDPOINT = f"{BOS_ENDPOINT}/{__name}" class BOSComponentException(BOSException): diff --git a/src/bos/reporter/components/state.py b/src/bos/reporter/components/state.py index aa5eaf13..c184d9ca 100644 --- a/src/bos/reporter/components/state.py +++ b/src/bos/reporter/components/state.py @@ -58,7 +58,7 @@ def patch_component(component, properties, session=None): the BOS API endpoint. """ session = session or requests_retry_session() - component_endpoint = '%s/%s' % (COMPONENT_ENDPOINT, component) + component_endpoint = f'{COMPONENT_ENDPOINT}/{component}' try: response = session.patch(component_endpoint, json=properties) except (ConnectionError, MaxRetryError) as ce: diff --git a/src/bos/reporter/proc_cmdline.py b/src/bos/reporter/proc_cmdline.py index fd9d6f0a..73b017a9 100644 --- a/src/bos/reporter/proc_cmdline.py +++ b/src/bos/reporter/proc_cmdline.py @@ -60,4 +60,4 @@ def get_value_from_proc_cmdline(key): except ValueError: # Single string values are not interesting to us continue - raise KeyError("Key '%s' was not discovered on '/proc/cmdline'" % key) + raise KeyError(f"Key '{key}' was not discovered on '/proc/cmdline'") diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 674384ce..bad62a2f 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -247,7 +247,7 @@ def patch_v2_components(): LOGGER.error("Unexpected data type %s", str(type(data))) return connexion.problem( status=400, title="Error parsing the data provided.", - detail="Unexpected data type {}".format(str(type(data)))) + detail=f"Unexpected data type {type(data).__name__}") def patch_v2_components_list(data): @@ -260,7 +260,7 @@ def patch_v2_components_list(data): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( status=404, title="Component not found.", - detail="Component {} could not be found".format(component_id)) + detail=f"Component {component_id} could not be found") components.append((component_id, component_data)) except Exception as err: LOGGER.error("Error loading component data: %s", exc_type_msg(err)) @@ -299,7 +299,7 @@ def patch_v2_components_dict(data): if component_id not in DB or not _is_valid_tenant_component(component_id): return connexion.problem( status=404, title="Component not found.", - detail="Component {} could not be found".format(component_id)) + detail=f"Component {component_id} could not be found") elif session: id_list = [component["id"] for component in get_v2_components_data( session=session, @@ -329,7 +329,7 @@ def get_v2_component(component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( status=404, title="Component not found.", - detail="Component {} could not be found".format(component_id)) + detail=f"Component {component_id} could not be found") component = DB.get(component_id) component = _set_status(component) del_timestamp(component) @@ -391,7 +391,7 @@ def patch_v2_component(component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( status=404, title="Component not found.", - detail="Component {} could not be found".format(component_id)) + detail=f"Component {component_id} could not be found") if "actual_state" in data and not validate_actual_state_change_is_allowed(component_id): LOGGER.warning("Not able to update actual state") return connexion.problem( @@ -431,7 +431,7 @@ def delete_v2_component(component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( status=404, title="Component not found.", - detail="Component {} could not be found".format(component_id)) + detail=f"Component {component_id} could not be found") return DB.delete(component_id), 204 diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index c0b55fbd..ff9bb7eb 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -75,7 +75,7 @@ def post_v2_session(): # noqa: E501 # Check that the template_name exists. session_template_response = get_v2_sessiontemplate(template_name) if isinstance(session_template_response, ConnexionResponse): - msg = "Session Template Name invalid: {}".format(template_name) + msg = f"Session Template Name invalid: {template_name}" LOGGER.error(msg) return msg, 400 session_template, _ = session_template_response @@ -93,7 +93,7 @@ def post_v2_session(): # noqa: E501 if session_key in DB: LOGGER.warning("v2 session named %s already exists", session.name) return connexion.problem( - detail="A session with the name {} already exists".format(session.name), + detail=f"A session with the name {session.name} already exists", status=409, title="Conflicting session name" ) @@ -151,7 +151,7 @@ def patch_v2_session(session_id): LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( status=404, title="Session could not found.", - detail="Session {} could not be found".format(session_id)) + detail=f"Session {session_id} could not be found") component = DB.patch(session_key, patch_data_json) return component, 200 @@ -172,7 +172,7 @@ def get_v2_session(session_id): # noqa: E501 LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( status=404, title="Session could not found.", - detail="Session {} could not be found".format(session_id)) + detail=f"Session {session_id} could not be found") session = DB.get(session_key) return session, 200 @@ -204,7 +204,7 @@ def delete_v2_session(session_id): # noqa: E501 LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( status=404, title="Session could not found.", - detail="Session {} could not be found".format(session_id)) + detail=f"Session {session_id} could not be found") if session_key in STATUS_DB: STATUS_DB.delete(session_key) return DB.delete(session_key), 204 @@ -251,7 +251,7 @@ def get_v2_session_status(session_id): # noqa: E501 LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( status=404, title="Session could not found.", - detail="Session {} could not be found".format(session_id)) + detail=f"Session {session_id} could not be found") session = DB.get(session_key) if session.get("status", {}).get("status") == "complete" and session_key in STATUS_DB: # If the session is complete and the status is saved, @@ -275,7 +275,7 @@ def save_v2_session_status(session_id): # noqa: E501 LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( status=404, title="Session could not found.", - detail="Session {} could not be found".format(session_id)) + detail=f"Session {session_id} could not be found") return STATUS_DB.put(session_key, _get_v2_session_status(session_key)), 200 @@ -386,7 +386,7 @@ def _get_v2_session_status(session_key, session=None): def _age_to_timestamp(age): delta = {} for interval in ['weeks', 'days', 'hours', 'minutes']: - result = re.search(r'(\d+)\w*{}'.format(interval[0]), age, re.IGNORECASE) + result = re.search(fr'(\d+)\w*{interval[0]}', age, re.IGNORECASE) if result: delta[interval] = int(result.groups()[0]) delta = timedelta(**delta) diff --git a/src/bos/server/controllers/v2/sessiontemplates.py b/src/bos/server/controllers/v2/sessiontemplates.py index 0c3aab70..3c20456b 100644 --- a/src/bos/server/controllers/v2/sessiontemplates.py +++ b/src/bos/server/controllers/v2/sessiontemplates.py @@ -148,7 +148,7 @@ def get_v2_sessiontemplate(session_template_id): LOGGER.warning("Session template not found: %s", session_template_id) return connexion.problem( status=404, title="Sessiontemplate could not found.", - detail="Sessiontemplate {} could not be found".format(session_template_id)) + detail=f"Sessiontemplate {session_template_id} could not be found") template = DB.get(template_key) return template, 200 @@ -178,7 +178,7 @@ def delete_v2_sessiontemplate(session_template_id): LOGGER.warning("Session template not found: %s", session_template_id) return connexion.problem( status=404, title="Sessiontemplate could not found.", - detail="Sessiontemplate {} could not be found".format(session_template_id)) + detail=f"Sessiontemplate {session_template_id} could not be found") return DB.delete(template_key), 204 @@ -196,7 +196,7 @@ def patch_v2_sessiontemplate(session_template_id): LOGGER.warning("Session template not found: %s", session_template_id) return connexion.problem( status=404, title="Sessiontemplate could not found.", - detail="Sessiontemplate {} could not be found".format(session_template_id)) + detail=f"Sessiontemplate {session_template_id} could not be found") if connexion.request.is_json: LOGGER.debug("connexion.request.is_json") From bd550ce0ff776762faf4b0d4136b26be89c86346 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 14:37:58 -0400 Subject: [PATCH 21/29] Linting: arguments-renamed --- src/bos/operators/filters/filters.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bos/operators/filters/filters.py b/src/bos/operators/filters/filters.py index fd6e0f6f..df51314d 100644 --- a/src/bos/operators/filters/filters.py +++ b/src/bos/operators/filters/filters.py @@ -120,8 +120,8 @@ def negated_match(*args, **kwargs): def _filter(self, components: List[dict]) -> List[dict]: return self.negated_filter._filter(components) - def _match(self, components: dict): - return self.negated_filter._match(components) + def _match(self, component: dict): + return self.negated_filter._match(component) class TimeSinceLastAction(LocalFilter): From 238b465b8f8c2df69ed7a33e6952c822d267dccf Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 15:20:38 -0400 Subject: [PATCH 22/29] Handle case where no path value is set in boot set in boot_image_metadata/factory.py --- CHANGELOG.md | 5 ++++- src/bos/operators/utils/boot_image_metadata/factory.py | 9 +++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23b1bb62..f0519ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## Unreleased +## [Unreleased] + +### Fixed +- Handle case where no path value is set in boot set in `boot_image_metadata/factory.py` ## [2.21.0] diff --git a/src/bos/operators/utils/boot_image_metadata/factory.py b/src/bos/operators/utils/boot_image_metadata/factory.py index 13a86982..03932cb8 100644 --- a/src/bos/operators/utils/boot_image_metadata/factory.py +++ b/src/bos/operators/utils/boot_image_metadata/factory.py @@ -43,7 +43,8 @@ def __init__(self, boot_set): def __call__(self): path_type = self.boot_set.get('type', None) - if path_type: - if path_type == 's3': - return S3BootImageMetaData(self.boot_set) - raise BootImageMetaDataUnknown(f"No BootImageMetaData class for type {path_type}") + if not path_type: + raise BootImageMetaDataUnknown(f"No path type set in boot set: {self.boot_set}") + if path_type == 's3': + return S3BootImageMetaData(self.boot_set) + raise BootImageMetaDataUnknown(f"No BootImageMetaData class for type {path_type}") From 9227631b3345b3f68910e3b5422b7d05e33c55b6 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 15:13:49 -0400 Subject: [PATCH 23/29] Raise exception when there is an error getting the service version --- CHANGELOG.md | 1 + api/openapi.yaml.in | 4 +++- src/bos/server/controllers/v2/base.py | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0519ae2..7f17613f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Handle case where no path value is set in boot set in `boot_image_metadata/factory.py` +- Raise exception when there is an error getting the service version ## [2.21.0] diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 8072cc77..5dfed717 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1321,6 +1321,8 @@ paths: type: array items: $ref: '#/components/schemas/Version' + 500: + $ref: '#/components/responses/InternalError' /v2: get: @@ -1335,7 +1337,7 @@ paths: 200: $ref: '#/components/responses/Version' 500: - $ref: '#/components/responses/BadRequest' + $ref: '#/components/responses/InternalError' /v2/healthz: get: summary: Get service health details diff --git a/src/bos/server/controllers/v2/base.py b/src/bos/server/controllers/v2/base.py index 8ff927bc..5cfda958 100644 --- a/src/bos/server/controllers/v2/base.py +++ b/src/bos/server/controllers/v2/base.py @@ -55,7 +55,8 @@ def calc_version(details): major, minor, patch = openapispec_map['info']['version'].split('.') return Version(major=major, minor=minor, patch=patch, links=links) except IOError as e: - LOGGER.debug('error opening "%s" file: %s', openapispec_f, exc_type_msg(e)) + LOGGER.exception('error opening "%s" file: %s', openapispec_f, exc_type_msg(e)) + raise def get_v2(): From 73833aef7842aa0feac6fb98328797a4ddb5ac1b Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 15:22:42 -0400 Subject: [PATCH 24/29] Update CHANGELOG to reflect linting --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f17613f..fb7ef00b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- Code linting (no functional changes) + ### Fixed - Handle case where no path value is set in boot set in `boot_image_metadata/factory.py` - Raise exception when there is an error getting the service version From ff0afe187ace8e2ea19706ef2280477e3c85c7b7 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 15 Jul 2024 16:44:17 -0400 Subject: [PATCH 25/29] Fix typo in set comprehension --- src/bos/operators/utils/clients/hsm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bos/operators/utils/clients/hsm.py b/src/bos/operators/utils/clients/hsm.py index b44e4dfe..f6cb3d95 100644 --- a/src/bos/operators/utils/clients/hsm.py +++ b/src/bos/operators/utils/clients/hsm.py @@ -74,8 +74,8 @@ def read_all_node_xnames(): LOGGER.error("Non-JSON response from HSM: %s", response.text) raise HWStateManagerException(jde) from jde try: - return {[component['ID'] for component in json_body['Components'] - if component.get('Type', None) == 'Node']} + return {component['ID'] for component in json_body['Components'] + if component.get('Type', None) == 'Node'} except KeyError as ke: LOGGER.error("Unexpected API response from HSM: %s", exc_type_msg(ke)) raise HWStateManagerException(ke) from ke From 4ba7413ea77e2b08aea80efbdb1349f793bcf633 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Wed, 17 Jul 2024 13:18:42 -0400 Subject: [PATCH 26/29] Correct error in method docstring in session_setup operator (cherry picked from commit 55aebc7cf25f0edf38b74ea6da8f2e07528c0fad) --- src/bos/operators/session_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 91e0b2ad..748e98d9 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -66,7 +66,7 @@ def _act(self, components): return components def _run(self) -> None: - """ A single pass of complete sessions """ + """ A single pass of pending sessions """ sessions = self._get_pending_sessions() if not sessions: return From 0ab910769f7b531428b52c695c9d445590f13473 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 22 Jul 2024 13:45:21 -0400 Subject: [PATCH 27/29] CASMCMS-9052: Add request timeouts to BOS reporter --- CHANGELOG.md | 5 ++++ src/bos/reporter/client.py | 25 +++----------------- src/bos/reporter/status_reporter/__main__.py | 18 +------------- 3 files changed, 9 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb7ef00b..64fb6b9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Add request timeouts to BOS reporter API calls ### Changed - Code linting (no functional changes) @@ -14,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Handle case where no path value is set in boot set in `boot_image_metadata/factory.py` - Raise exception when there is an error getting the service version +### Removed +- Removed redundant `duration_to_timedelta` function definition from BOS reporter source. + ## [2.21.0] ### Fixed diff --git a/src/bos/reporter/client.py b/src/bos/reporter/client.py index a5facc10..acce256d 100644 --- a/src/bos/reporter/client.py +++ b/src/bos/reporter/client.py @@ -30,10 +30,9 @@ import logging import subprocess import time +from functools import partial -import requests -from requests.adapters import HTTPAdapter -from requests.packages.urllib3.util.retry import Retry +from bos.common.utils import requests_retry_session as common_requests_retry_session from . import PROTOCOL @@ -74,22 +73,4 @@ def get_auth_token(path='/opt/cray/auth-utils/bin/get-auth-token'): time.sleep(2) -def requests_retry_session(retries=10, connect=10, backoff_factor=0.5, - status_forcelist=(500, 502, 503, 504), - session=None): - """ - Returns a session with retries built into it. - """ - session = session or requests.Session() - retry = Retry( - total=retries, - read=retries, - connect=retries, - backoff_factor=backoff_factor, - status_forcelist=status_forcelist, - ) - adapter = HTTPAdapter(max_retries=retry) - session.mount(PROTOCOL, adapter) - auth_token = get_auth_token() - session.headers.update({'Authorization': f'Bearer {auth_token}'}) - return session +requests_retry_session = partial(common_requests_retry_session, protocol=PROTOCOL) diff --git a/src/bos/reporter/status_reporter/__main__.py b/src/bos/reporter/status_reporter/__main__.py index 3a7d2dea..3d260246 100644 --- a/src/bos/reporter/status_reporter/__main__.py +++ b/src/bos/reporter/status_reporter/__main__.py @@ -28,7 +28,7 @@ import datetime from time import sleep -from bos.common.utils import exc_type_msg +from bos.common.utils import duration_to_timedelta, exc_type_msg from bos.reporter.client import requests_retry_session from bos.reporter.node_identity import read_identity from bos.reporter.components.state import report_state, BOSComponentException, UnknownComponent @@ -93,22 +93,6 @@ def report_state_until_success(component): return -def duration_to_timedelta(timestamp: str): - """ - Converts a to a timedelta object. - """ - # Calculate the corresponding multiplier for each time value - seconds_table = {'s': 1, - 'm': 60, - 'h': 60 * 60, - 'd': 60 * 60 * 24, - 'w': 60 * 60 * 24 * 7} - timeval, durationval = TIME_DURATION_PATTERN.search(timestamp).groups() - timeval = float(timeval) - seconds = timeval * seconds_table[durationval] - return datetime.timedelta(seconds = seconds) - - def main(): """ Read the Boot Artifact ID from the /proc/cmdline and report it to the BOS From 5bec250cf48e9016f5b1a742884897794e3ef856 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Tue, 23 Jul 2024 12:17:07 -0400 Subject: [PATCH 28/29] CASMTRIAGE-7147: Create max_component_batch_size option to limit number of nodes operators work on at a time (cherry picked from commit 7997503711b40b74615f6c0c9ef7e8dc3528da04) --- CHANGELOG.md | 2 + api/openapi.yaml.in | 6 +++ src/bos/operators/base.py | 41 ++++++++++++++++++- src/bos/operators/discovery.py | 5 ++- src/bos/operators/session_setup.py | 15 +++---- src/bos/operators/status.py | 9 ++++ .../operators/utils/clients/bos/options.py | 3 ++ src/bos/server/controllers/v2/options.py | 1 + 8 files changed, 72 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64fb6b9a..fbd228b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add request timeouts to BOS reporter API calls ### Changed +- Create new BOS v2 `max_component_batch_size` option to limit number of components a BOS operator + will work on at once. - Code linting (no functional changes) ### Fixed diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 5dfed717..5be2f42d 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1047,6 +1047,12 @@ components: example: 1 minimum: 0 maximum: 1048576 + max_component_batch_size: + type: integer + description: The maximum number of components that a BOS operator will process at once. 0 means no limit. + example: 1000 + minimum: 0 + maximum: 131071 additionalProperties: true requestBodies: diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 248dbf6f..dc530cb2 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -27,11 +27,12 @@ """ from abc import ABC, abstractmethod +import itertools import logging import threading import os import time -from typing import List, NoReturn, Type +from typing import Generator, List, NoReturn, Type from bos.common.utils import exc_type_msg from bos.common.values import Status @@ -74,6 +75,7 @@ class BaseOperator(ABC): def __init__(self) -> NoReturn: self.bos_client = BOSClient() + self.__max_batch_size = 0 @property @abstractmethod @@ -108,6 +110,23 @@ def run(self) -> NoReturn: LOGGER.exception('Unhandled exception getting polling frequency: %s', e) time.sleep(5) # A small sleep for when exceptions getting the polling frequency + @property + def max_batch_size(self) -> int: + max_batch_size = options.max_component_batch_size + if max_batch_size != self.__max_batch_size: + LOGGER.info("max_component_batch_size option set to %d", max_batch_size) + self.__max_batch_size = max_batch_size + return max_batch_size + + def _chunk_components(self, components: List[dict]) -> Generator[List[dict], None, None]: + """ + Break up the components into groups of no more than max_batch_size nodes, + and yield each group in turn. + If the max size is set to 0, just yield the entire list. + """ + for chunk in chunk_components(components, self.max_batch_size): + yield chunk + def _run(self) -> None: """ A single pass of detecting and acting on components """ components = self._get_components() @@ -115,6 +134,14 @@ def _run(self) -> None: LOGGER.debug('Found 0 components that require action') return LOGGER.info('Found %d components that require action', len(components)) + for chunk in self._chunk_components(components): + self._run_on_chunk(chunk) + + def _run_on_chunk(self, components: List[dict]) -> None: + """ + Acts on a chunk of components + """ + LOGGER.debug("Processing %d components", len(components)) # Only check for failed components if we track retries for this operator if self.retry_attempt_field: components = self._handle_failed_components(components) @@ -259,6 +286,18 @@ def _update_database_for_failure(self, components: List[dict]) -> None: self.bos_client.components.update_components(data) +def chunk_components(components: List[dict], + max_batch_size: int) -> Generator[List[dict], None, None]: + """ + Break up the components into groups of no more than max_batch_size nodes, + and yield each group in turn. + If the max size is set to 0, just yield the entire list. + """ + chunk_size = max_batch_size if max_batch_size > 0 else len(components) + for chunk in itertools.batched(components, chunk_size): + yield chunk + + def _update_log_level() -> None: """ Updates the current logging level base on the value in the options database """ try: diff --git a/src/bos/operators/discovery.py b/src/bos/operators/discovery.py index 6f5ec77a..14f1c7eb 100644 --- a/src/bos/operators/discovery.py +++ b/src/bos/operators/discovery.py @@ -78,8 +78,9 @@ def _run(self) -> None: LOGGER.info("No new component(s) discovered.") return LOGGER.info("%s new component(s) from HSM.", len(components_to_add)) - self.bos_client.components.put_components(components_to_add) - LOGGER.info("%s new component(s) added to BOS!", len(components_to_add)) + for chunk in self._chunk_components(components_to_add): + self.bos_client.components.put_components(chunk) + LOGGER.info("%s new component(s) added to BOS!", len(chunk)) @property def bos_components(self) -> Set[str]: diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 748e98d9..fe59c9d1 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -27,7 +27,7 @@ from typing import Set from botocore.exceptions import ClientError -from bos.operators.base import BaseOperator, main +from bos.operators.base import BaseOperator, main, chunk_components from bos.operators.filters.filters import HSMState from bos.operators.utils.clients.hsm import Inventory from bos.operators.utils.clients.s3 import S3Object, S3ObjectNotFound @@ -74,7 +74,7 @@ def _run(self) -> None: inventory_cache = Inventory() for data in sessions: session = Session(data, inventory_cache, self.bos_client) - session.setup() + session.setup(self.max_batch_size) def _get_pending_sessions(self): return self.bos_client.sessions.get_sessions(status='pending') @@ -108,15 +108,15 @@ def template(self): self.tenant) return self._template - def setup(self): + def setup(self, max_batch_size: int): try: - component_ids = self._setup_components() + component_ids = self._setup_components(max_batch_size) except SessionSetupException as err: self._mark_failed(str(err)) else: self._mark_running(component_ids) - def _setup_components(self): + def _setup_components(self, max_batch_size: int): all_component_ids = [] data = [] stage = self.session_data.get("stage", False) @@ -138,8 +138,9 @@ def _setup_components(self): raise SessionSetupException(err) from err # No exception raised by previous block self._log(LOGGER.info, 'Found %d components that require updates', len(data)) - self._log(LOGGER.debug, f'Updated components: {data}') - self.bos_client.components.update_components(data) + for chunk in chunk_components(data, max_batch_size): + self._log(LOGGER.debug, f'Updated components: {chunk}') + self.bos_client.components.update_components(chunk) return list(set(all_component_ids)) def _get_boot_set_component_list(self, boot_set) -> Set[str]: diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index 529be818..e60ffd7a 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -74,6 +74,15 @@ def _run(self) -> None: if not components: LOGGER.debug('No enabled components found') return + LOGGER.debug('Found %d components that require action', len(components)) + for chunk in self._chunk_components(components): + self._run_on_chunk(chunk) + + def _run_on_chunk(self, components) -> None: + """ + Acts on a chunk of components + """ + LOGGER.debug("Processing %d components", len(components)) component_ids = [component['id'] for component in components] power_states = node_to_powerstate(component_ids) cfs_states = self._get_cfs_components() diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/operators/utils/clients/bos/options.py index ed15522d..3440facb 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/operators/utils/clients/bos/options.py @@ -116,6 +116,9 @@ def component_actual_state_ttl(self): def default_retry_policy(self): return self.get_option('default_retry_policy', int, 3) + @property + def max_component_batch_size(self): + return self.get_option('max_component_batch_size', int, 2800) options = Options() diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index fab768b1..35d98fae 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -49,6 +49,7 @@ 'max_power_off_wait_time': 300, 'polling_frequency': 15, 'default_retry_policy': 3, + 'max_component_batch_size': 2800 } From 6c02e1f973ed19af5678bc9e4914f782812027cc Mon Sep 17 00:00:00 2001 From: Mitch Harding Date: Tue, 23 Jul 2024 12:39:09 -0400 Subject: [PATCH 29/29] Update CHANGELOG.md Release 2.22.0 for CSM 1.6 --- CHANGELOG.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbd228b8..8de405bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [2.22.0] ### Added - Add request timeouts to BOS reporter API calls - -### Changed - Create new BOS v2 `max_component_batch_size` option to limit number of components a BOS operator will work on at once. + +### Changed - Code linting (no functional changes) ### Fixed