From ab78213dfad6b18ea0a01d3923712dc9ee652fa5 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 16 Dec 2024 15:43:30 -0500 Subject: [PATCH 1/4] CASMCMS-9225: Add paging to component list requests; use context managers for all requests --- .gitignore | 2 + CHANGELOG.md | 6 + api/openapi.yaml.in | 14 + constraints.txt.in | 2 +- requirements.txt | 2 +- src/bos/common/clients/__init__.py | 0 src/bos/common/clients/api_client.py | 62 ++++ src/bos/common/clients/bos/__init__.py | 25 ++ src/bos/common/clients/bos/base.py | 121 +++++++ src/bos/common/clients/bos/client.py | 43 +++ .../clients/bos/components.py | 20 +- .../utils => common}/clients/bos/options.py | 25 +- .../clients/bos/session_templates.py | 4 +- .../utils => common}/clients/bos/sessions.py | 9 +- .../clients/bss}/__init__.py | 3 +- src/bos/common/clients/bss/base.py | 41 +++ src/bos/common/clients/bss/boot_parameters.py | 76 ++++ src/bos/common/clients/bss/client.py | 33 ++ src/bos/common/clients/cfs/__init__.py | 24 ++ src/bos/common/clients/cfs/base.py | 71 ++++ .../clients/cfs/client.py} | 19 +- src/bos/common/clients/cfs/components.py | 112 ++++++ src/bos/common/clients/endpoints/__init__.py | 30 ++ .../common/clients/endpoints/base_endpoint.py | 41 +++ .../endpoints/base_generic_endpoint.py | 119 +++++++ .../clients/endpoints/base_raw_endpoint.py | 41 +++ src/bos/common/clients/endpoints/defs.py | 42 +++ .../common/clients/endpoints/exceptions.py | 34 ++ .../endpoints/request_error_handler.py | 116 ++++++ .../common/clients/endpoints/response_data.py | 53 +++ src/bos/common/clients/hsm/__init__.py | 27 ++ src/bos/common/clients/hsm/base.py | 60 ++++ src/bos/common/clients/hsm/client.py | 43 +++ src/bos/common/clients/hsm/exceptions.py | 33 ++ src/bos/common/clients/hsm/groups.py | 28 ++ src/bos/common/clients/hsm/inventory.py | 124 +++++++ src/bos/common/clients/hsm/partitions.py | 28 ++ .../common/clients/hsm/state_components.py | 99 ++++++ src/bos/common/clients/ims/__init__.py | 26 ++ src/bos/common/clients/ims/base.py | 44 +++ src/bos/common/clients/ims/client.py | 33 ++ src/bos/common/clients/ims/defs.py | 44 +++ src/bos/common/clients/ims/exceptions.py | 34 ++ src/bos/common/clients/ims/images.py | 88 +++++ src/bos/common/clients/ims/utils.py | 63 ++++ src/bos/common/clients/pcs/__init__.py | 27 ++ src/bos/common/clients/pcs/base.py | 60 ++++ src/bos/common/clients/pcs/client.py | 38 ++ src/bos/common/clients/pcs/exceptions.py | 56 +++ src/bos/common/clients/pcs/power_status.py | 124 +++++++ src/bos/common/clients/pcs/transitions.py | 149 ++++++++ .../{operators/utils => common}/clients/s3.py | 33 +- src/bos/common/tenant_utils.py | 53 +-- src/bos/common/utils.py | 96 ++++- src/bos/operators/actual_state_cleanup.py | 21 +- src/bos/operators/base.py | 168 ++++++--- src/bos/operators/configuration.py | 11 +- src/bos/operators/discovery.py | 29 +- src/bos/operators/filters/base.py | 20 +- src/bos/operators/filters/filters.py | 92 +++-- src/bos/operators/power_off_forceful.py | 18 +- src/bos/operators/power_off_graceful.py | 10 +- src/bos/operators/power_on.py | 87 +++-- src/bos/operators/session_cleanup.py | 17 +- src/bos/operators/session_completion.py | 25 +- src/bos/operators/session_setup.py | 145 +++++--- src/bos/operators/status.py | 74 ++-- .../utils/boot_image_metadata/factory.py | 10 +- .../s3_boot_image_metadata.py | 14 +- src/bos/operators/utils/clients/bos/base.py | 197 ----------- .../utils/clients/bos/sessions_status.py | 63 ---- src/bos/operators/utils/clients/bss.py | 87 ----- src/bos/operators/utils/clients/cfs.py | 140 -------- src/bos/operators/utils/clients/hsm.py | 239 ------------- src/bos/operators/utils/clients/ims.py | 179 ---------- src/bos/operators/utils/clients/pcs.py | 291 --------------- src/bos/operators/utils/liveness/__main__.py | 9 +- src/bos/server/__main__.py | 5 +- src/bos/server/controllers/base.py | 6 +- src/bos/server/controllers/utils.py | 10 +- src/bos/server/controllers/v2/base.py | 5 +- .../controllers/v2/boot_set/artifacts.py | 9 +- .../server/controllers/v2/boot_set/defs.py | 6 +- src/bos/server/controllers/v2/boot_set/ims.py | 31 +- src/bos/server/controllers/v2/components.py | 333 +++++++++++------- src/bos/server/controllers/v2/healthz.py | 2 +- src/bos/server/controllers/v2/options.py | 18 +- src/bos/server/controllers/v2/sessions.py | 167 +++++---- .../server/controllers/v2/sessiontemplates.py | 102 +++--- src/bos/server/dbs/boot_artifacts.py | 23 +- src/bos/server/migrations/__main__.py | 4 +- src/bos/server/migrations/db.py | 26 +- src/bos/server/migrations/sanitize.py | 69 ++-- src/bos/server/migrations/validate.py | 22 +- src/bos/server/redis_db_utils.py | 58 ++- src/bos/server/schema.py | 5 +- src/bos/server/utils.py | 7 +- 97 files changed, 3535 insertions(+), 1919 deletions(-) create mode 100644 src/bos/common/clients/__init__.py create mode 100644 src/bos/common/clients/api_client.py create mode 100644 src/bos/common/clients/bos/__init__.py create mode 100644 src/bos/common/clients/bos/base.py create mode 100644 src/bos/common/clients/bos/client.py rename src/bos/{operators/utils => common}/clients/bos/components.py (69%) rename src/bos/{operators/utils => common}/clients/bos/options.py (75%) rename src/bos/{operators/utils => common}/clients/bos/session_templates.py (92%) rename src/bos/{operators/utils => common}/clients/bos/sessions.py (84%) rename src/bos/{operators/utils/clients => common/clients/bss}/__init__.py (92%) create mode 100644 src/bos/common/clients/bss/base.py create mode 100644 src/bos/common/clients/bss/boot_parameters.py create mode 100644 src/bos/common/clients/bss/client.py create mode 100644 src/bos/common/clients/cfs/__init__.py create mode 100644 src/bos/common/clients/cfs/base.py rename src/bos/{operators/utils/clients/bos/__init__.py => common/clients/cfs/client.py} (72%) create mode 100644 src/bos/common/clients/cfs/components.py create mode 100644 src/bos/common/clients/endpoints/__init__.py create mode 100644 src/bos/common/clients/endpoints/base_endpoint.py create mode 100644 src/bos/common/clients/endpoints/base_generic_endpoint.py create mode 100644 src/bos/common/clients/endpoints/base_raw_endpoint.py create mode 100644 src/bos/common/clients/endpoints/defs.py create mode 100644 src/bos/common/clients/endpoints/exceptions.py create mode 100644 src/bos/common/clients/endpoints/request_error_handler.py create mode 100644 src/bos/common/clients/endpoints/response_data.py create mode 100644 src/bos/common/clients/hsm/__init__.py create mode 100644 src/bos/common/clients/hsm/base.py create mode 100644 src/bos/common/clients/hsm/client.py create mode 100644 src/bos/common/clients/hsm/exceptions.py create mode 100644 src/bos/common/clients/hsm/groups.py create mode 100644 src/bos/common/clients/hsm/inventory.py create mode 100644 src/bos/common/clients/hsm/partitions.py create mode 100644 src/bos/common/clients/hsm/state_components.py create mode 100644 src/bos/common/clients/ims/__init__.py create mode 100644 src/bos/common/clients/ims/base.py create mode 100644 src/bos/common/clients/ims/client.py create mode 100644 src/bos/common/clients/ims/defs.py create mode 100644 src/bos/common/clients/ims/exceptions.py create mode 100644 src/bos/common/clients/ims/images.py create mode 100644 src/bos/common/clients/ims/utils.py create mode 100644 src/bos/common/clients/pcs/__init__.py create mode 100644 src/bos/common/clients/pcs/base.py create mode 100644 src/bos/common/clients/pcs/client.py create mode 100644 src/bos/common/clients/pcs/exceptions.py create mode 100644 src/bos/common/clients/pcs/power_status.py create mode 100644 src/bos/common/clients/pcs/transitions.py rename src/bos/{operators/utils => common}/clients/s3.py (91%) delete mode 100644 src/bos/operators/utils/clients/bos/base.py delete mode 100644 src/bos/operators/utils/clients/bos/sessions_status.py delete mode 100644 src/bos/operators/utils/clients/bss.py delete mode 100644 src/bos/operators/utils/clients/cfs.py delete mode 100644 src/bos/operators/utils/clients/hsm.py delete mode 100644 src/bos/operators/utils/clients/ims.py delete mode 100644 src/bos/operators/utils/clients/pcs.py diff --git a/.gitignore b/.gitignore index b1028314..cafd55ac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ results .project .settings .version +.venv +.idea .pydevproject ansible/bos_deploy.retry .tox diff --git a/CHANGELOG.md b/CHANGELOG.md index c46cde36..b597b885 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Added basic paging ability for `GET` requests for `components`. + +### Changed +- Modified operators to use paging when requesting BOS components, using a page size equal to the `max_components_batch_size` option. +- Put all requests code into context managers -- this includes the HTTP adapters, the sessions, and the request responses. ## [2.31.0] - 2024-11-01 ### Removed diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 31475590..5a2612b4 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1691,6 +1691,20 @@ paths: in: query description: |- Retrieve the Components with the given status. + - name: start_after_id + schema: + $ref: '#/components/schemas/V2ComponentId' + in: query + description: |- + Begin listing Components after the specified ID. Used for paging. + - name: page_size + schema: + type: integer + minimum: 0 + maximum: 1048576 + in: query + description: |- + Maximum number of Components to include in response. Used for paging. description: |- Retrieve the full collection of Components in the form of a ComponentArray. Full results can also be filtered by query diff --git a/constraints.txt.in b/constraints.txt.in index f7434180..295b5cf5 100644 --- a/constraints.txt.in +++ b/constraints.txt.in @@ -37,7 +37,7 @@ PyYAML>=6.0.1,<6.1 redis>=5.0,<5.1 requests>=2.28.2,<2.29 requests-oauthlib>=1.3.1,<1.4 -requests-retry-session>=0.1,<0.2 +requests-retry-session>=2.0,<2.1 retrying>=1.3.4,<1.4 rsa>=4.9,<4.10 s3transfer>=0.6.2,<0.7 diff --git a/requirements.txt b/requirements.txt index e4118e7a..f01eff6d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ python-dateutil PyYAML redis requests -requests-retry-session +requests-retry-session>=0.2.2 urllib3 # The purpose of this file is to contain python runtime requirements diff --git a/src/bos/common/clients/__init__.py b/src/bos/common/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/bos/common/clients/api_client.py b/src/bos/common/clients/api_client.py new file mode 100644 index 00000000..40a1990b --- /dev/null +++ b/src/bos/common/clients/api_client.py @@ -0,0 +1,62 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +from typing import Type, TypeVar + +from bos.common.clients.endpoints import BaseGenericEndpoint +from bos.common.utils import RetrySessionManager + +ClientEndpoint = TypeVar('ClientEndpoint', bound=BaseGenericEndpoint) + + +class APIClient(RetrySessionManager, ABC): + """ + As a subclass of RetrySessionManager, this class can be used as a context manager, + and will have a requests session available as self.requests_session + + This context manager is used to provide API endpoints, via subclassing. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._endpoint_values: dict[Type[ClientEndpoint], ClientEndpoint] = {} + + def get_endpoint(self, + endpoint_type: Type[ClientEndpoint]) -> ClientEndpoint: + """ + Endpoints are created only as needed, and passed the manager retry session. + """ + if endpoint_type not in self._endpoint_values: + self._endpoint_values[endpoint_type] = endpoint_type( + self.requests_session) + return self._endpoint_values[endpoint_type] + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool | None: + """ + The only cleanup we need to do when exiting the context manager is to clear out + our list of API clients. Our call to super().__exit__ will take care of closing + out the underlying request session. + """ + self._endpoint_values.clear() + return super().__exit__(exc_type, exc_val, exc_tb) diff --git a/src/bos/common/clients/bos/__init__.py b/src/bos/common/clients/bos/__init__.py new file mode 100644 index 00000000..2e6580aa --- /dev/null +++ b/src/bos/common/clients/bos/__init__.py @@ -0,0 +1,25 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from .client import BOSClient diff --git a/src/bos/common/clients/bos/base.py b/src/bos/common/clients/bos/base.py new file mode 100644 index 00000000..a73bec0d --- /dev/null +++ b/src/bos/common/clients/bos/base.py @@ -0,0 +1,121 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +import logging + +from bos.common.clients.endpoints import BaseEndpoint +from bos.common.tenant_utils import get_new_tenant_header +from bos.common.utils import PROTOCOL + +LOGGER = logging.getLogger(__name__) + +API_VERSION = 'v2' +SERVICE_NAME = 'cray-bos' +BASE_BOS_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{API_VERSION}" + + +class BaseBosEndpoint(BaseEndpoint, ABC): + """ + This base class provides generic access to the BOS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = BASE_BOS_ENDPOINT + + +class BaseBosNonTenantAwareEndpoint(BaseBosEndpoint, ABC): + """ + This base class provides generic access to the BOS API for non-tenant-aware endpoints + The individual endpoint needs to be overridden for a specific endpoint. + """ + + def get_item(self, item_id): + """Get information for a single BOS item""" + return self.get(uri=item_id) + + def get_items(self, **kwargs): + """Get information for all BOS items""" + return self.get(params=kwargs) + + def update_item(self, item_id, data): + """Update information for a single BOS item""" + return self.patch(uri=item_id, json=data) + + def update_items(self, data): + """Update information for multiple BOS items""" + return self.patch(json=data) + + def put_items(self, data): + """Put information for multiple BOS Items""" + return self.put(json=data) + + def delete_items(self, **kwargs): + """Delete information for multiple BOS items""" + return self.delete(params=kwargs) + + +class BaseBosTenantAwareEndpoint(BaseBosEndpoint, ABC): + """ + This base class provides generic access to the BOS API for tenant aware endpoints. + The individual endpoint needs to be overridden for a specific endpoint. + """ + + def get_item(self, item_id, tenant): + """Get information for a single BOS item""" + return self.get(uri=item_id, headers=get_new_tenant_header(tenant)) + + def get_items(self, **kwargs): + """Get information for all BOS items""" + headers = None + if "tenant" in kwargs: + tenant = kwargs.pop("tenant") + headers = get_new_tenant_header(tenant) + return self.get(params=kwargs, headers=headers) + + def update_item(self, item_id, tenant, data): + """Update information for a single BOS item""" + return self.patch(uri=item_id, + json=data, + headers=get_new_tenant_header(tenant)) + + def update_items(self, tenant, data): + """Update information for multiple BOS items""" + return self.patch(json=data, headers=get_new_tenant_header(tenant)) + + def post_item(self, item_id, tenant, data=None): + """Post information for a single BOS item""" + return self.post(uri=item_id, + json=data, + headers=get_new_tenant_header(tenant)) + + def put_items(self, tenant, data): + """Put information for multiple BOS items""" + return self.put(json=data, headers=get_new_tenant_header(tenant)) + + def delete_items(self, **kwargs): + """Delete information for multiple BOS items""" + headers = None + if "tenant" in kwargs: + tenant = kwargs.pop("tenant") + headers = get_new_tenant_header(tenant) + return self.delete(params=kwargs, headers=headers) diff --git a/src/bos/common/clients/bos/client.py b/src/bos/common/clients/bos/client.py new file mode 100644 index 00000000..8c411b14 --- /dev/null +++ b/src/bos/common/clients/bos/client.py @@ -0,0 +1,43 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from bos.common.clients.api_client import APIClient + +from .components import ComponentEndpoint +from .sessions import SessionEndpoint +from .session_templates import SessionTemplateEndpoint + + +class BOSClient(APIClient): + + @property + def components(self) -> ComponentEndpoint: + return self.get_endpoint(ComponentEndpoint) + + @property + def sessions(self) -> SessionEndpoint: + return self.get_endpoint(SessionEndpoint) + + @property + def session_templates(self) -> SessionTemplateEndpoint: + return self.get_endpoint(SessionTemplateEndpoint) diff --git a/src/bos/operators/utils/clients/bos/components.py b/src/bos/common/clients/bos/components.py similarity index 69% rename from src/bos/operators/utils/clients/bos/components.py rename to src/bos/common/clients/bos/components.py index 2e17f7d4..e8681839 100644 --- a/src/bos/operators/utils/clients/bos/components.py +++ b/src/bos/common/clients/bos/components.py @@ -23,19 +23,31 @@ # import logging -from .base import BaseBosEndpoint +from .base import BaseBosNonTenantAwareEndpoint +from .options import options -LOGGER = logging.getLogger('bos.operators.utils.clients.bos.components') +LOGGER = logging.getLogger(__name__) -class ComponentEndpoint(BaseBosEndpoint): +class ComponentEndpoint(BaseBosNonTenantAwareEndpoint): ENDPOINT = __name__.lower().rsplit('.', maxsplit=1)[-1] def get_component(self, component_id): return self.get_item(component_id) def get_components(self, **kwargs): - return self.get_items(**kwargs) + page_size = options.max_component_batch_size + if page_size == 0: + return self.get_items(**kwargs) + next_page = self.get_items(page_size=page_size, **kwargs) + results = next_page + while len(next_page) == page_size: + last_id = next_page[-1]["id"] + next_page = self.get_items(page_size=page_size, + start_after_id=last_id, + **kwargs) + results.extend(next_page) + return results def update_component(self, component_id, data): return self.update_item(component_id, data) diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/common/clients/bos/options.py similarity index 75% rename from src/bos/operators/utils/clients/bos/options.py rename to src/bos/common/clients/bos/options.py index c6e4a890..168e2582 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/common/clients/bos/options.py @@ -23,17 +23,20 @@ # import logging import json +from typing import Optional -from requests.exceptions import HTTPError, ConnectionError +import requests +from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError as RequestsConnectionError from urllib3.exceptions import MaxRetryError from bos.common.options import OptionsCache -from bos.common.utils import exc_type_msg, requests_retry_session -from bos.operators.utils.clients.bos.base import BASE_ENDPOINT +from bos.common.utils import exc_type_msg, retry_session_get +from bos.common.clients.bos.base import BASE_BOS_ENDPOINT -LOGGER = logging.getLogger('bos.operators.utils.clients.bos.options') +LOGGER = logging.getLogger(__name__) __name = __name__.lower().rsplit('.', maxsplit=1)[-1] -ENDPOINT = f"{BASE_ENDPOINT}/{__name}" +ENDPOINT = f"{BASE_BOS_ENDPOINT}/{__name}" class Options(OptionsCache): @@ -43,15 +46,15 @@ class Options(OptionsCache): This caches the options so that frequent use of these options do not all result in network calls. """ - def _get_options(self) -> dict: + + def _get_options(self, session: Optional[requests.Session] = None) -> dict: """Retrieves the current options from the BOS api""" - session = requests_retry_session() LOGGER.debug("GET %s", ENDPOINT) try: - response = session.get(ENDPOINT) - response.raise_for_status() - return json.loads(response.text) - except (ConnectionError, MaxRetryError) as e: + with retry_session_get(ENDPOINT, session=session) as response: + response.raise_for_status() + return json.loads(response.text) + except (RequestsConnectionError, MaxRetryError) as e: LOGGER.error("Unable to connect to BOS: %s", exc_type_msg(e)) except HTTPError as e: LOGGER.error("Unexpected response from BOS: %s", exc_type_msg(e)) diff --git a/src/bos/operators/utils/clients/bos/session_templates.py b/src/bos/common/clients/bos/session_templates.py similarity index 92% rename from src/bos/operators/utils/clients/bos/session_templates.py rename to src/bos/common/clients/bos/session_templates.py index 842bf639..09c5cec4 100644 --- a/src/bos/operators/utils/clients/bos/session_templates.py +++ b/src/bos/common/clients/bos/session_templates.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -25,7 +25,7 @@ from .base import BaseBosTenantAwareEndpoint -LOGGER = logging.getLogger('bos.operators.utils.clients.bos.session_templates') +LOGGER = logging.getLogger(__name__) class SessionTemplateEndpoint(BaseBosTenantAwareEndpoint): diff --git a/src/bos/operators/utils/clients/bos/sessions.py b/src/bos/common/clients/bos/sessions.py similarity index 84% rename from src/bos/operators/utils/clients/bos/sessions.py rename to src/bos/common/clients/bos/sessions.py index 3d958e1f..6d5b7c17 100644 --- a/src/bos/operators/utils/clients/bos/sessions.py +++ b/src/bos/common/clients/bos/sessions.py @@ -25,7 +25,7 @@ from .base import BaseBosTenantAwareEndpoint -LOGGER = logging.getLogger('bos.operators.utils.clients.bos.sessions') +LOGGER = logging.getLogger(__name__) class SessionEndpoint(BaseBosTenantAwareEndpoint): @@ -42,3 +42,10 @@ def update_session(self, session_id, tenant, data): def delete_sessions(self, **kwargs): return self.delete_items(**kwargs) + + def post_session_status(self, session_id, tenant): + """ + Post information for a single BOS Session status. + This basically saves the BOS Session status to the database. + """ + return self.post_item(f'{session_id}/status', tenant) diff --git a/src/bos/operators/utils/clients/__init__.py b/src/bos/common/clients/bss/__init__.py similarity index 92% rename from src/bos/operators/utils/clients/__init__.py rename to src/bos/common/clients/bss/__init__.py index 21bcdd30..7bf62fa7 100644 --- a/src/bos/operators/utils/clients/__init__.py +++ b/src/bos/common/clients/bss/__init__.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -21,3 +21,4 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +from .client import BSSClient diff --git a/src/bos/common/clients/bss/base.py b/src/bos/common/clients/bss/base.py new file mode 100644 index 00000000..73b6096c --- /dev/null +++ b/src/bos/common/clients/bss/base.py @@ -0,0 +1,41 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +import logging + +from bos.common.clients.endpoints import BaseRawEndpoint +from bos.common.utils import PROTOCOL + +LOGGER = logging.getLogger(__name__) + +SERVICE_NAME = 'cray-bss' +ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/boot/v1" + + +class BaseBssEndpoint(BaseRawEndpoint, ABC): + """ + This base class provides generic access to the BSS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = ENDPOINT diff --git a/src/bos/common/clients/bss/boot_parameters.py b/src/bos/common/clients/bss/boot_parameters.py new file mode 100644 index 00000000..280f47c9 --- /dev/null +++ b/src/bos/common/clients/bss/boot_parameters.py @@ -0,0 +1,76 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +import logging + +from .base import BaseBssEndpoint + +LOGGER = logging.getLogger(__name__) + + +class BootParametersEndpoint(BaseBssEndpoint): + ENDPOINT = 'bootparameters' + + def set_bss(self, node_set, kernel_params, kernel, initrd) -> str: + ''' + Tell the Boot Script Service (BSS) which boot artifacts are associated + with each node. + + Currently, this is biased towards 'hosts' (i.e. xnames) rather than + NIDS. + + Args: + node_set (set): A list of nodes to assign the boot artifacts to + kernel_params (string): Kernel parameters to assign to the node + kernel (string): The kernel to assign to the node + initrd (string): The initrd to assign to the node + session (requests Session instance): An existing session to use + + Returns: + The 'bss-referral-token' value from the header of the response from BSS. + + Raises: + KeyError -- 'bss-referral-token' not found in header + requests.exceptions.HTTPError -- An HTTP error encountered while + communicating with the + Hardware State Manager + Exception -- called with empty node_set + ''' + if not node_set: + # Cannot simply return if no nodes are specified, as this function + # is intended to return the response object from BSS. + # Accordingly, an Exception is raised. + raise Exception("set_bss called with empty node_set") + + LOGGER.info("Params: %s", kernel_params) + + # Assignment payload + payload = { + "hosts": list(node_set), + "params": kernel_params, + "kernel": kernel, + "initrd": initrd + } + + return self.put(json=payload, + verify=False).headers['bss-referral-token'] diff --git a/src/bos/common/clients/bss/client.py b/src/bos/common/clients/bss/client.py new file mode 100644 index 00000000..85d58cbe --- /dev/null +++ b/src/bos/common/clients/bss/client.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from bos.common.clients.api_client import APIClient + +from .boot_parameters import BootParametersEndpoint + + +class BSSClient(APIClient): + + @property + def boot_parameters(self) -> BootParametersEndpoint: + return self.get_endpoint(BootParametersEndpoint) diff --git a/src/bos/common/clients/cfs/__init__.py b/src/bos/common/clients/cfs/__init__.py new file mode 100644 index 00000000..bb123f0b --- /dev/null +++ b/src/bos/common/clients/cfs/__init__.py @@ -0,0 +1,24 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from .client import CFSClient diff --git a/src/bos/common/clients/cfs/base.py b/src/bos/common/clients/cfs/base.py new file mode 100644 index 00000000..bd14804a --- /dev/null +++ b/src/bos/common/clients/cfs/base.py @@ -0,0 +1,71 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +import logging + +from bos.common.clients.endpoints import BaseEndpoint +from bos.common.utils import PROTOCOL + +LOGGER = logging.getLogger(__name__) + +SERVICE_NAME = 'cray-cfs-api' +BASE_CFS_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/v3" + + +class BaseCfsEndpoint(BaseEndpoint, ABC): + """ + This base class provides generic access to the CFS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = BASE_CFS_ENDPOINT + + def get_items(self, **kwargs): + """Get information for all CFS items""" + return self.get(params=kwargs) + + def update_items(self, data): + """Update information for multiple CFS items""" + return self.patch(json=data) + + +class BasePagedCfsEndpoint(BaseCfsEndpoint, ABC): + """ + This base class provides generic access to the CFS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + ITEM_FIELD_NAME = '' + + def get_items(self, **kwargs): + """Get information for all CFS items""" + item_list = [] + while kwargs is not None: + response_json = super().get_items(**kwargs) + new_items = response_json[self.ITEM_FIELD_NAME] + LOGGER.debug("Query returned %d %ss", len(new_items), + self.ITEM_FIELD_NAME) + item_list.extend(new_items) + kwargs = response_json["next"] + LOGGER.debug("Returning %d %ss from CFS", len(item_list), + self.ITEM_FIELD_NAME) + return item_list diff --git a/src/bos/operators/utils/clients/bos/__init__.py b/src/bos/common/clients/cfs/client.py similarity index 72% rename from src/bos/operators/utils/clients/bos/__init__.py rename to src/bos/common/clients/cfs/client.py index 04c4e94f..68f05bb0 100644 --- a/src/bos/operators/utils/clients/bos/__init__.py +++ b/src/bos/common/clients/cfs/client.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -21,16 +21,17 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +from bos.common.clients.api_client import APIClient +from bos.common.clients.bos.options import options + from .components import ComponentEndpoint -from .sessions import SessionEndpoint -from .session_templates import SessionTemplateEndpoint -from .sessions_status import SessionStatusEndpoint -class BOSClient: +class CFSClient(APIClient): def __init__(self): - self.components = ComponentEndpoint() - self.sessions = SessionEndpoint() - self.session_status = SessionStatusEndpoint() - self.session_templates = SessionTemplateEndpoint() + super().__init__(read_timeout=options.cfs_read_timeout) + + @property + def components(self) -> ComponentEndpoint: + return self.get_endpoint(ComponentEndpoint) diff --git a/src/bos/common/clients/cfs/components.py b/src/bos/common/clients/cfs/components.py new file mode 100644 index 00000000..94d8cf74 --- /dev/null +++ b/src/bos/common/clients/cfs/components.py @@ -0,0 +1,112 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from collections import defaultdict +import logging + +from .base import BasePagedCfsEndpoint + +LOGGER = logging.getLogger(__name__) + +GET_BATCH_SIZE = 200 +PATCH_BATCH_SIZE = 1000 + + +class ComponentEndpoint(BasePagedCfsEndpoint): + ENDPOINT = 'components' + + def get_components(self, **kwargs): + return self.get_items(**kwargs) + + def patch_components(self, data): + return self.update_items(data) + + def get_components_from_id_list(self, id_list): + if not id_list: + LOGGER.warning( + "get_components_from_id_list called without IDs; returning without action." + ) + return [] + LOGGER.debug("get_components_from_id_list called with %d IDs", + len(id_list)) + component_list = [] + while id_list: + next_batch = id_list[:GET_BATCH_SIZE] + next_comps = self.get_components(ids=','.join(next_batch)) + component_list.extend(next_comps) + id_list = id_list[GET_BATCH_SIZE:] + LOGGER.debug( + "get_components_from_id_list returning a total of %d components from CFS", + len(component_list)) + return component_list + + def patch_desired_config(self, + node_ids, + desired_config, + enabled: bool = False, + tags=None, + clear_state: bool = False): + if not node_ids: + LOGGER.warning( + "patch_desired_config called without IDs; returning without action." + ) + return + LOGGER.debug( + "patch_desired_config called on %d IDs with desired_config=%s enabled=%s tags=%s" + " clear_state=%s", len(node_ids), desired_config, enabled, tags, + clear_state) + node_patch = { + 'enabled': enabled, + 'desired_config': desired_config, + 'tags': tags if tags else {} + } + data = {"patch": node_patch, "filters": {}} + if clear_state: + node_patch['state'] = [] + while node_ids: + data["filters"]["ids"] = ','.join(node_ids[:PATCH_BATCH_SIZE]) + self.patch_components(data) + node_ids = node_ids[PATCH_BATCH_SIZE:] + + def set_cfs(self, components, enabled: bool, clear_state: bool = False): + if not components: + LOGGER.warning( + "set_cfs called without components; returning without action.") + return + LOGGER.debug( + "set_cfs called on %d components with enabled=%s clear_state=%s", + len(components), enabled, clear_state) + configurations = defaultdict(list) + for component in components: + config_name = component.get('desired_state', + {}).get('configuration', '') + bos_session = component.get('session') + key = (config_name, bos_session) + configurations[key].append(component['id']) + for key, ids in configurations.items(): + config_name, bos_session = key + self.patch_desired_config(ids, + config_name, + enabled=enabled, + tags={'bos_session': bos_session}, + clear_state=clear_state) diff --git a/src/bos/common/clients/endpoints/__init__.py b/src/bos/common/clients/endpoints/__init__.py new file mode 100644 index 00000000..5ea4171a --- /dev/null +++ b/src/bos/common/clients/endpoints/__init__.py @@ -0,0 +1,30 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from .base_endpoint import BaseEndpoint +from .base_generic_endpoint import BaseGenericEndpoint, RequestErrorHandler +from .base_raw_endpoint import BaseRawEndpoint +from .defs import JsonData, RequestData, RequestsMethod +from .exceptions import ApiResponseError +from .response_data import ResponseData diff --git a/src/bos/common/clients/endpoints/base_endpoint.py b/src/bos/common/clients/endpoints/base_endpoint.py new file mode 100644 index 00000000..21517fb5 --- /dev/null +++ b/src/bos/common/clients/endpoints/base_endpoint.py @@ -0,0 +1,41 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC + +import requests + +from .base_generic_endpoint import BaseGenericEndpoint +from .defs import JsonData +from .response_data import ResponseData + + +class BaseEndpoint(BaseGenericEndpoint[JsonData], ABC): + """ + This base class provides generic access to an API where the only part of the response + that is returned is the body. + """ + + @classmethod + def format_response(cls, response: requests.Response) -> JsonData: + return ResponseData.from_response(response).body diff --git a/src/bos/common/clients/endpoints/base_generic_endpoint.py b/src/bos/common/clients/endpoints/base_generic_endpoint.py new file mode 100644 index 00000000..b68e3705 --- /dev/null +++ b/src/bos/common/clients/endpoints/base_generic_endpoint.py @@ -0,0 +1,119 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC, abstractmethod +import logging +from typing import Generic, TypeVar + +import requests + +from .defs import RequestData, RequestsMethod +from .exceptions import ApiResponseError +from .request_error_handler import BaseRequestErrorHandler, RequestErrorHandler + +LOGGER = logging.getLogger(__name__) + +RequestReturnT = TypeVar('RequestReturnT') + + +class BaseGenericEndpoint(ABC, Generic[RequestReturnT]): + """ + This base class provides generic access to an API endpoint. + RequestReturnT represents the type of data this API will return. + Most often this will be the Json data from the response body, but in some + cases (like with BSS), we are after something else. + + Exceptions are handled by a separate class, since different API clients + may want to handle these differently. + """ + BASE_ENDPOINT: str = '' + ENDPOINT: str = '' + error_handler: BaseRequestErrorHandler = RequestErrorHandler + + def __init__(self, session: requests.Session): + super().__init__() + self.session = session + + @classmethod + @abstractmethod + def format_response(cls, response: requests.Response) -> RequestReturnT: + ... + + @classmethod + def base_url(cls) -> str: + return f"{cls.BASE_ENDPOINT}/{cls.ENDPOINT}" + + @classmethod + def url(cls, uri: str) -> str: + base_url = cls.base_url() + if not uri: + return base_url + if uri[0] == '/' or base_url[-1] == '/': + return f"{base_url}{uri}" + return f"{base_url}/{uri}" + + def request(self, + method: RequestsMethod, + /, + *, + uri: str = "", + **kwargs) -> RequestReturnT: + url = self.url(uri) + LOGGER.debug("%s %s (kwargs=%s)", method.__name__.upper(), url, kwargs) + try: + return self._request(method, url, **kwargs) + except Exception as err: + self.error_handler.handle_exception( + err, + RequestData(method_name=method.__name__.upper(), + url=url, + request_options=kwargs)) + + @classmethod + def _request(cls, method: RequestsMethod, url: str, /, + **kwargs) -> RequestReturnT: + """Make API request""" + with method(url, **kwargs) as response: + if not response.ok: + raise ApiResponseError(response=response) + return cls.format_response(response) + + def delete(self, **kwargs) -> RequestReturnT: + """Delete request""" + return self.request(self.session.delete, **kwargs) + + def get(self, **kwargs) -> RequestReturnT: + """Get request""" + return self.request(self.session.get, **kwargs) + + def patch(self, **kwargs) -> RequestReturnT: + """Patch request""" + return self.request(self.session.patch, **kwargs) + + def post(self, **kwargs) -> RequestReturnT: + """Post request""" + return self.request(self.session.post, **kwargs) + + def put(self, **kwargs) -> RequestReturnT: + """Put request""" + return self.request(self.session.put, **kwargs) diff --git a/src/bos/common/clients/endpoints/base_raw_endpoint.py b/src/bos/common/clients/endpoints/base_raw_endpoint.py new file mode 100644 index 00000000..b9149ff6 --- /dev/null +++ b/src/bos/common/clients/endpoints/base_raw_endpoint.py @@ -0,0 +1,41 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC + +import requests + +from .base_generic_endpoint import BaseGenericEndpoint +from .response_data import ResponseData + + +class BaseRawEndpoint(BaseGenericEndpoint[ResponseData], ABC): + """ + This base class provides generic access to an API. + In this case, an assortment of response data is returned up, rather than + just the response body. + """ + + @classmethod + def format_response(cls, response: requests.Response) -> ResponseData: + return ResponseData.from_response(response) diff --git a/src/bos/common/clients/endpoints/defs.py b/src/bos/common/clients/endpoints/defs.py new file mode 100644 index 00000000..8b74cd27 --- /dev/null +++ b/src/bos/common/clients/endpoints/defs.py @@ -0,0 +1,42 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from typing import Any, Callable, ContextManager, NamedTuple + +import requests + +type JsonData = bool | str | None | int | float | list[JsonData] | dict[str, JsonData] +type JsonDict = dict[str, JsonData] +type JsonList = list[JsonData] + +type RequestsMethod = Callable[..., ContextManager[requests.Response]] + +class RequestData(NamedTuple): + """ + This class encapsulates data about an API request. + It is passed into the exception handler, so that it is able to + include information about the request in its logic and error messages. + """ + method_name: str + url: str + request_options: dict[str, Any] diff --git a/src/bos/common/clients/endpoints/exceptions.py b/src/bos/common/clients/endpoints/exceptions.py new file mode 100644 index 00000000..8b57247c --- /dev/null +++ b/src/bos/common/clients/endpoints/exceptions.py @@ -0,0 +1,34 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +import requests + +from .response_data import ResponseData + + +class ApiResponseError(Exception): + """Raised when API response has non-ok status""" + + def __init__(self, *args, response: requests.Response, **kwargs): + super().__init__(*args, **kwargs) + self.response_data = ResponseData.from_response(response) diff --git a/src/bos/common/clients/endpoints/request_error_handler.py b/src/bos/common/clients/endpoints/request_error_handler.py new file mode 100644 index 00000000..4de38993 --- /dev/null +++ b/src/bos/common/clients/endpoints/request_error_handler.py @@ -0,0 +1,116 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC, abstractmethod +from json import JSONDecodeError +import logging +from typing import NoReturn + +from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError as RequestsConnectionError +from urllib3.exceptions import MaxRetryError + +from bos.common.utils import compact_response_text, exc_type_msg + +from .defs import RequestData, RequestsMethod +from .exceptions import ApiResponseError + +LOGGER = logging.getLogger(__name__) + + +class BaseRequestErrorHandler(ABC): + """ + The abstract base class for request error handlers that will be used by an API endpoint. + """ + @classmethod + @abstractmethod + def handle_exception(cls, err: Exception, + request_data: RequestData) -> NoReturn: + ... + + +class RequestErrorHandler(BaseRequestErrorHandler): + """ + The default request error handler used by API endpoints. + """ + @classmethod + def handle_api_response_error(cls, err: ApiResponseError, + request_data: RequestData) -> NoReturn: + msg = (f"Non-2XX response ({err.response.status_code}) to " + f"{request_data.method_name} {request_data.url}; " + f"{err.response.reason} " + f"{compact_response_text(err.response.text)}") + LOGGER.error(msg) + raise ApiResponseError(msg, response=err.response) from err + + @classmethod + def handle_connection_error(cls, err: RequestsConnectionError, + request_data: RequestData) -> NoReturn: + LOGGER.error("%s %s: Unable to connect: %s", request_data.method_name, + request_data.url, exc_type_msg(err)) + raise err + + @classmethod + def handle_http_error(cls, err: HTTPError, + request_data: RequestData) -> NoReturn: + LOGGER.error("%s %s: Unexpected response: %s", + request_data.method_name, request_data.url, + exc_type_msg(err)) + raise err + + @classmethod + def handle_json_decode_error(cls, err: JSONDecodeError, + request_data: RequestData) -> NoReturn: + LOGGER.error("%s %s: Non-JSON response: %s", request_data.method_name, + request_data.url, exc_type_msg(err)) + raise err + + @classmethod + def handle_max_retry_error(cls, err: MaxRetryError, + request_data: RequestData) -> NoReturn: + LOGGER.error("%s %s: Request failed after retries: %s", + request_data.method_name, request_data.url, + exc_type_msg(err)) + raise err + + @classmethod + def default(cls, err: Exception, request_data: RequestData) -> NoReturn: + LOGGER.error("%s %s: Unexpected exception: %s", + request_data.method_name, request_data.url, + exc_type_msg(err)) + raise err + + @classmethod + def handle_exception(cls, err: Exception, + request_data: RequestData) -> NoReturn: + if isinstance(err, ApiResponseError): + cls.handle_api_response_error(err, request_data) + if isinstance(err, RequestsConnectionError): + cls.handle_connection_error(err, request_data) + if isinstance(err, HTTPError): + cls.handle_http_error(err, request_data) + if isinstance(err, JSONDecodeError): + cls.handle_json_decode_error(err, request_data) + if isinstance(err, MaxRetryError): + cls.handle_max_retry_error(err, request_data) + cls.default(err, request_data) diff --git a/src/bos/common/clients/endpoints/response_data.py b/src/bos/common/clients/endpoints/response_data.py new file mode 100644 index 00000000..d7577f64 --- /dev/null +++ b/src/bos/common/clients/endpoints/response_data.py @@ -0,0 +1,53 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +import json +from typing import NamedTuple, Self + +import requests + +from .defs import JsonData, JsonDict + + +class ResponseData(NamedTuple): + """ + Encapsulates data from a response to an API request. This allows the + response itself to be cleaned up when its context manager exits. + """ + headers: JsonDict + ok: bool + reason: str + status_code: int + text: bytes | None + + @property + def body(self) -> JsonData: + return json.loads(self.text) if self.text else None + + @classmethod + def from_response(cls, resp: requests.Response) -> Self: + return cls(headers=resp.headers, + ok=resp.ok, + reason=resp.reason, + status_code=resp.status_code, + text=resp.text) diff --git a/src/bos/common/clients/hsm/__init__.py b/src/bos/common/clients/hsm/__init__.py new file mode 100644 index 00000000..fab88214 --- /dev/null +++ b/src/bos/common/clients/hsm/__init__.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from .client import HSMClient +from .exceptions import HWStateManagerException +from .inventory import Inventory diff --git a/src/bos/common/clients/hsm/base.py b/src/bos/common/clients/hsm/base.py new file mode 100644 index 00000000..1cc99bb9 --- /dev/null +++ b/src/bos/common/clients/hsm/base.py @@ -0,0 +1,60 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +from json import JSONDecodeError + +from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError as RequestsConnectionError +from urllib3.exceptions import MaxRetryError + +from bos.common.clients.endpoints import ApiResponseError, BaseEndpoint, JsonData, RequestsMethod +from bos.common.utils import PROTOCOL + +from .exceptions import HWStateManagerException + +SERVICE_NAME = 'cray-smd' +ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/hsm/v2/" + + +class BaseHsmEndpoint(BaseEndpoint, ABC): + """ + This base class provides generic access to the HSM API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = ENDPOINT + + def request(self, + method: RequestsMethod, + /, + *, + uri: str = "", + **kwargs) -> JsonData: + try: + return super().request(method, uri=uri, **kwargs) + except (ApiResponseError, RequestsConnectionError, HTTPError, + JSONDecodeError, MaxRetryError) as err: + raise HWStateManagerException(err) from err + + def list(self, params=None): + return self.get(params=params) diff --git a/src/bos/common/clients/hsm/client.py b/src/bos/common/clients/hsm/client.py new file mode 100644 index 00000000..eba076a1 --- /dev/null +++ b/src/bos/common/clients/hsm/client.py @@ -0,0 +1,43 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from bos.common.clients.api_client import APIClient + +from .groups import GroupsEndpoint +from .partitions import PartitionsEndpoint +from .state_components import StateComponentsEndpoint + + +class HSMClient(APIClient): + + @property + def groups(self) -> GroupsEndpoint: + return self.get_endpoint(GroupsEndpoint) + + @property + def partitions(self) -> PartitionsEndpoint: + return self.get_endpoint(PartitionsEndpoint) + + @property + def state_components(self) -> StateComponentsEndpoint: + return self.get_endpoint(StateComponentsEndpoint) diff --git a/src/bos/common/clients/hsm/exceptions.py b/src/bos/common/clients/hsm/exceptions.py new file mode 100644 index 00000000..cf4cf97d --- /dev/null +++ b/src/bos/common/clients/hsm/exceptions.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + + +class HWStateManagerException(Exception): + """ + An error unique to interacting with the HWStateManager service; + should the service be unable to fulfill a given request (timeout, + no components, service 503s, etc.); this exception is raised. It is + intended to be further subclassed for more specific kinds of errors + in the future should they arise. + """ diff --git a/src/bos/common/clients/hsm/groups.py b/src/bos/common/clients/hsm/groups.py new file mode 100644 index 00000000..44fa75e4 --- /dev/null +++ b/src/bos/common/clients/hsm/groups.py @@ -0,0 +1,28 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from .base import BaseHsmEndpoint + + +class GroupsEndpoint(BaseHsmEndpoint): + ENDPOINT = 'groups' diff --git a/src/bos/common/clients/hsm/inventory.py b/src/bos/common/clients/hsm/inventory.py new file mode 100644 index 00000000..60879bc9 --- /dev/null +++ b/src/bos/common/clients/hsm/inventory.py @@ -0,0 +1,124 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from contextlib import AbstractContextManager +from collections import defaultdict +import logging +from types import TracebackType +from typing import Optional, Type + +from .client import HSMClient + +LOGGER = logging.getLogger(__name__) + + +class Inventory(AbstractContextManager): + """ + Inventory handles the generation of a hardware inventory in a similar manner to how the + dynamic inventory is generated for CFS. To reduce the number of calls to HSM, everything is + cached for repeated checks, stored both as overall inventory and separate group types to allow + use in finding BOS's base list of nodes, and lazily loaded to prevent extra calls when no limit + is used. + """ + + def __init__(self, partition=None): + super().__init__() + self._partition = partition # Can be specified to limit to roles/components query + self._inventory = None + self._groups = None + self._partitions = None + self._roles = None + self._hsm_client = None + + @property + def hsm_client(self) -> HSMClient: + if self._hsm_client is None: + self._hsm_client = HSMClient() + return self._hsm_client + + def __exit__(self, exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> bool | None: + if self._hsm_client is None: + return False + client = self._hsm_client + self._hsm_client = None + return client.__exit__(exc_type, exc_val, exc_tb) + + @property + def groups(self): + if self._groups is None: + data = self.hsm_client.groups.list() + groups = {} + for group in data: + groups[group['label']] = set( + group.get('members', {}).get('ids', [])) + self._groups = groups + return self._groups + + @property + def partitions(self): + if self._partitions is None: + data = self.hsm_client.partitions.list() + partitions = {} + for partition in data: + partitions[partition['name']] = set( + partition.get('members', {}).get('ids', [])) + self._partitions = partitions + return self._partitions + + @property + def roles(self): + if self._roles is None: + params = {} + if self._partition: + params['partition'] = self._partition + data = self.hsm_client.state_components.list(params=params) + roles = defaultdict(set) + for component in data['Components']: + role = '' + if 'Role' in component: + role = str(component['Role']) + roles[role].add(component['ID']) + if 'SubRole' in component: + subrole = role + '_' + str(component['SubRole']) + roles[subrole].add(component['ID']) + self._roles = roles + return self._roles + + @property + def inventory(self): + if self._inventory is None: + inventory = {} + inventory.update(self.groups) + inventory.update(self.partitions) + inventory.update(self.roles) + self._inventory = inventory + LOGGER.info(self._inventory) + return self._inventory + + def __contains__(self, key): + return key in self.inventory + + def __getitem__(self, key): + return self.inventory[key] diff --git a/src/bos/common/clients/hsm/partitions.py b/src/bos/common/clients/hsm/partitions.py new file mode 100644 index 00000000..b06b8cc1 --- /dev/null +++ b/src/bos/common/clients/hsm/partitions.py @@ -0,0 +1,28 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from .base import BaseHsmEndpoint + + +class PartitionsEndpoint(BaseHsmEndpoint): + ENDPOINT = 'partitions' diff --git a/src/bos/common/clients/hsm/state_components.py b/src/bos/common/clients/hsm/state_components.py new file mode 100644 index 00000000..2c72561e --- /dev/null +++ b/src/bos/common/clients/hsm/state_components.py @@ -0,0 +1,99 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +import logging + +from bos.common.utils import exc_type_msg + +from .base import BaseHsmEndpoint +from .exceptions import HWStateManagerException + +LOGGER = logging.getLogger(__name__) + + +class StateComponentsEndpoint(BaseHsmEndpoint): + ENDPOINT = 'State/Components' + + def read_all_node_xnames(self): + """ + Queries HSM for the full set of xname components that + have been discovered; return these as a set. + """ + json_body = self.get() + try: + return { + component['ID'] + for component in json_body['Components'] + if component.get('Type', None) == 'Node' + } + except KeyError as ke: + LOGGER.error("Unexpected API response from HSM: %s", + exc_type_msg(ke)) + raise HWStateManagerException(ke) from ke + + def get_components(self, node_list, enabled=None) -> dict[str, list[dict]]: + """ + Get information for all list components HSM + + :return the HSM components + :rtype Dictionary containing a 'Components' key whose value is a list + containing each component, where each component is itself represented by a + dictionary. + + Here is an example of the returned values. + { + "Components": [ + { + "ID": "x3000c0s19b1n0", + "Type": "Node", + "State": "Ready", + "Flag": "OK", + "Enabled": true, + "Role": "Compute", + "NID": 1, + "NetType": "Sling", + "Arch": "X86", + "Class": "River" + }, + { + "ID": "x3000c0s19b2n0", + "Type": "Node", + "State": "Ready", + "Flag": "OK", + "Enabled": true, + "Role": "Compute", + "NID": 1, + "NetType": "Sling", + "Arch": "X86", + "Class": "River" + } + ] + } + """ + if not node_list: + LOGGER.warning("hsm.get_components called with empty node list") + return {'Components': []} + payload = {'ComponentIDs': node_list} + if enabled is not None: + payload['enabled'] = [str(enabled)] + return self.post(uri="Query", json=payload) diff --git a/src/bos/common/clients/ims/__init__.py b/src/bos/common/clients/ims/__init__.py new file mode 100644 index 00000000..44735be8 --- /dev/null +++ b/src/bos/common/clients/ims/__init__.py @@ -0,0 +1,26 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from .client import IMSClient +from .exceptions import ImageNotFound, TagFailure +from .utils import get_ims_id_from_s3_url, get_arch_from_image_data diff --git a/src/bos/common/clients/ims/base.py b/src/bos/common/clients/ims/base.py new file mode 100644 index 00000000..14c3f88f --- /dev/null +++ b/src/bos/common/clients/ims/base.py @@ -0,0 +1,44 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC + +from bos.common.clients.endpoints import BaseEndpoint + +from .defs import BASE_ENDPOINT as BASE_IMS_ENDPOINT + + +class BaseImsEndpoint(BaseEndpoint, ABC): + """ + This base class provides generic access to the IMS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = BASE_IMS_ENDPOINT + + def get_item(self, item_id: str): + """Get information for a single IMS item""" + return self.get(uri=item_id) + + def update_item(self, item_id: str, data): + """Update information for a single IMS item""" + return self.patch(uri=item_id, json=data) diff --git a/src/bos/common/clients/ims/client.py b/src/bos/common/clients/ims/client.py new file mode 100644 index 00000000..cbc5bd3d --- /dev/null +++ b/src/bos/common/clients/ims/client.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from bos.common.clients.api_client import APIClient + +from .images import ImagesEndpoint + + +class IMSClient(APIClient): + + @property + def images(self) -> ImagesEndpoint: + return self.get_endpoint(ImagesEndpoint) diff --git a/src/bos/common/clients/ims/defs.py b/src/bos/common/clients/ims/defs.py new file mode 100644 index 00000000..e547880e --- /dev/null +++ b/src/bos/common/clients/ims/defs.py @@ -0,0 +1,44 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +import logging +import re + +from bos.common.utils import PROTOCOL + +LOGGER = logging.getLogger(__name__) + +SERVICE_NAME = 'cray-ims' +IMS_VERSION = 'v3' +BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{IMS_VERSION}" + +IMS_TAG_OPERATIONS = ['set', 'remove'] + +# Making minimal assumptions about the IMS ID itself, this pattern just makes sure that the +# S3 key is some string, then a /, then at least one more character. +IMS_S3_KEY_RE = r'^([^/]+)/.+' +IMS_S3_KEY_RE_PROG = re.compile(IMS_S3_KEY_RE) + +# If an IMS image does not have the arch field, default to x86_64 for purposes of +# backward-compatibility +DEFAULT_IMS_IMAGE_ARCH = 'x86_64' diff --git a/src/bos/common/clients/ims/exceptions.py b/src/bos/common/clients/ims/exceptions.py new file mode 100644 index 00000000..0befaba1 --- /dev/null +++ b/src/bos/common/clients/ims/exceptions.py @@ -0,0 +1,34 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +class TagFailure(Exception): + pass + + +class ImageNotFound(Exception): + """ + Raised if querying IMS for an image and it is not found + """ + + def __init__(self, image_id: str): + super().__init__(f"IMS image id '{image_id}' does not exist in IMS") diff --git a/src/bos/common/clients/ims/images.py b/src/bos/common/clients/ims/images.py new file mode 100644 index 00000000..2fc467b9 --- /dev/null +++ b/src/bos/common/clients/ims/images.py @@ -0,0 +1,88 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from typing import NoReturn + +from bos.common.clients.endpoints import ApiResponseError, RequestData, RequestErrorHandler, \ + RequestsMethod + +from .base import BaseImsEndpoint +from .defs import IMS_TAG_OPERATIONS, LOGGER +from .exceptions import ImageNotFound, TagFailure + + +class ImsImageRequestErrorHandler(RequestErrorHandler): + + @classmethod + def handle_api_response_error(cls, err: ApiResponseError, + request_data: RequestData) -> NoReturn: + if err.response_data.status_code == 404: + # If it's not found, we just log it as a warning, because we may be + # okay with that -- that will be for the caller to decide + LOGGER.warning("%s %s: 404 response", request_data.method_name, + request_data.url) + image_id = request_data.url.split('/')[-1] + raise ImageNotFound(image_id) from err + super().handle_api_response_error(err, request_data) + + +class ImagesEndpoint(BaseImsEndpoint): + ENDPOINT = 'images' + error_handler = ImsImageRequestErrorHandler + + def get_image(self, image_id: str) -> dict: + return self.get_item(image_id) + + def patch_image(self, image_id: str, data) -> None: + self.update_item(image_id, data) + + def tag_image(self, + image_id: str, + operation: str, + key: str, + value: str = None) -> None: + if operation not in IMS_TAG_OPERATIONS: + msg = f"{operation} not valid. Expecting one of {IMS_TAG_OPERATIONS}" + LOGGER.error(msg) + raise TagFailure(msg) + + if not key: + msg = f"key must exist: {key}" + LOGGER.error(msg) + raise TagFailure(msg) + + if value: + LOGGER.debug("Patching image %s %sing key: %s value: %s", image_id, + operation, key, value) + else: + LOGGER.debug("Patching image %s %sing key: %s", image_id, + operation, key) + + data = { + "metadata": { + "operation": operation, + "key": key, + "value": value + } + } + self.patch_image(image_id=image_id, data=data) diff --git a/src/bos/common/clients/ims/utils.py b/src/bos/common/clients/ims/utils.py new file mode 100644 index 00000000..6e6acfe0 --- /dev/null +++ b/src/bos/common/clients/ims/utils.py @@ -0,0 +1,63 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from bos.common.utils import exc_type_msg +from bos.common.clients.s3 import S3Url + +from .defs import IMS_S3_KEY_RE_PROG, DEFAULT_IMS_IMAGE_ARCH, LOGGER + + +def get_ims_id_from_s3_url(s3_url: S3Url) -> str | None: + """ + If the s3_url matches the expected format of an IMS image path, then return the IMS image ID. + Otherwise return None. + """ + try: + return IMS_S3_KEY_RE_PROG.match(s3_url.key).group(1) + except (AttributeError, IndexError): + return None + + +def get_arch_from_image_data(image_data: dict) -> str: + """ + Returns the value of the 'arch' field in the image data + If it is not present, logs a warning and returns the default value + """ + try: + arch = image_data['arch'] + except KeyError: + LOGGER.warning( + "Defaulting to '%s' because 'arch' not set in IMS image data: %s", + DEFAULT_IMS_IMAGE_ARCH, image_data) + return DEFAULT_IMS_IMAGE_ARCH + except Exception as err: + LOGGER.error("Unexpected error parsing IMS image data (%s): %s", + exc_type_msg(err), image_data) + raise + if arch: + return arch + LOGGER.warning( + "Defaulting to '%s' because 'arch' set to null value in IMS image data: %s", + DEFAULT_IMS_IMAGE_ARCH, image_data) + return DEFAULT_IMS_IMAGE_ARCH diff --git a/src/bos/common/clients/pcs/__init__.py b/src/bos/common/clients/pcs/__init__.py new file mode 100644 index 00000000..4f475dcf --- /dev/null +++ b/src/bos/common/clients/pcs/__init__.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from .client import PCSClient +from .exceptions import (PowerControlException, PowerControlSyntaxException, + PowerControlTimeoutException, + PowerControlComponentsEmptyException) diff --git a/src/bos/common/clients/pcs/base.py b/src/bos/common/clients/pcs/base.py new file mode 100644 index 00000000..c9434be0 --- /dev/null +++ b/src/bos/common/clients/pcs/base.py @@ -0,0 +1,60 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +from json import JSONDecodeError +import logging + +from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError as RequestsConnectionError +from urllib3.exceptions import MaxRetryError + +from bos.common.clients.endpoints import ApiResponseError, BaseEndpoint, JsonData, RequestsMethod +from bos.common.utils import PROTOCOL + +from .exceptions import PowerControlException + +LOGGER = logging.getLogger(__name__) + +SERVICE_NAME = 'cray-power-control' +ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}" + + +class BasePcsEndpoint(BaseEndpoint, ABC): + """ + This base class provides generic access to the PCS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = ENDPOINT + + def request(self, + method: RequestsMethod, + /, + *, + uri: str = "", + **kwargs) -> JsonData: + try: + return super().request(method, uri=uri, **kwargs) + except (ApiResponseError, RequestsConnectionError, HTTPError, + JSONDecodeError, MaxRetryError) as err: + raise PowerControlException(err) from err diff --git a/src/bos/common/clients/pcs/client.py b/src/bos/common/clients/pcs/client.py new file mode 100644 index 00000000..c15e2c7c --- /dev/null +++ b/src/bos/common/clients/pcs/client.py @@ -0,0 +1,38 @@ +# +# MIT License +# +# (C) Copyright 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from bos.common.clients.api_client import APIClient + +from .power_status import PowerStatusEndpoint +from .transitions import TransitionsEndpoint + + +class PCSClient(APIClient): + + @property + def power_status(self) -> PowerStatusEndpoint: + return self.get_endpoint(PowerStatusEndpoint) + + @property + def transitions(self) -> TransitionsEndpoint: + return self.get_endpoint(TransitionsEndpoint) diff --git a/src/bos/common/clients/pcs/exceptions.py b/src/bos/common/clients/pcs/exceptions.py new file mode 100644 index 00000000..e6bbbd3b --- /dev/null +++ b/src/bos/common/clients/pcs/exceptions.py @@ -0,0 +1,56 @@ +# +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + + +class PowerControlException(Exception): + """ + Interaction with PCS resulted in a known failure. + """ + + +class PowerControlSyntaxException(Exception): + """ + A class of error raised when interacting with PCS in an unsupported way. + """ + + +class PowerControlTimeoutException(PowerControlException): + """ + Raised when a call to PowerControl exceeded total time to complete. + """ + + +class PowerControlComponentsEmptyException(Exception): + """ + Raised when one of the PCS utility functions that requires a non-empty + list of components is passed an empty component list. This will only + happen in the case of a programming bug. + + This exception is not raised for functions that require a node list + but that are able to return a sensible object to the caller that + indicates nothing has been done. For example, the status function. + This exception is instead used for functions that will fail if they run + with an empty node list, but which cannot return an appropriate + "no-op" value to the caller. + """ diff --git a/src/bos/common/clients/pcs/power_status.py b/src/bos/common/clients/pcs/power_status.py new file mode 100644 index 00000000..54f13377 --- /dev/null +++ b/src/bos/common/clients/pcs/power_status.py @@ -0,0 +1,124 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from collections import defaultdict +import logging + +from .base import BasePcsEndpoint + +LOGGER = logging.getLogger(__name__) + + +class PowerStatusEndpoint(BasePcsEndpoint): + ENDPOINT = 'power-status' + + def query(self, + xname=None, + power_state_filter=None, + management_state_filter=None): + """ + This is the one to one implementation to the underlying power control get query. + For reasons of compatibility with existing calls into older power control APIs, + existing functions call into this function to preserve the existing functionality + already implemented. + + Users may specify one of three filters, and a power_status_all (PCS defined schema) + is returned. Users may elect to use a previously generated session in order to query + the results. If not, the default requests retry session will be generated. + + Per the spec, a power_status_all is returned. power_status_all is an array of power + statuses. + """ + params = {} + if xname: + params['xname'] = xname + if power_state_filter: + assert power_state_filter.lower() in set( + ['on', 'off', 'undefined']) + params['powerStateFilter'] = power_state_filter.lower() + if management_state_filter: + assert management_state_filter in set(['available', 'unavailable']) + params['managementStateFilter'] = management_state_filter.lower() + # PCS added the POST option for this endpoint in app version 2.3.0 + # (chart versions 2.0.8 and 2.1.5) + return self.post(json=params) + + def status(self, nodes, **kwargs): + """ + For a given iterable of nodes, represented by xnames, query PCS for + the power status. Return a dictionary of nodes that have + been bucketed by status. + + Args: + nodes (list): Nodes to get status for + session (session object): An already instantiated session + kwargs: Any additional args used for filtering when calling _power_status. + This can be useful if you want to limit your search to only available or unavailable + nodes, and allows a more future-proof way of handling arguments to PCS as a catch-all + parameter. + + Returns: + status_dict (dict): Keys are different states; values are a literal set of nodes. + Nodes with errors associated with them are saved with the error value as a + status key. + + Raises: + PowerControlException: Any non-nominal response from PCS. + JSONDecodeError: Error decoding the PCS response + """ + status_bucket = defaultdict(set) + if not nodes: + LOGGER.warning( + "status called without nodes; returning without action.") + return status_bucket + power_status_all = self.query(xname=list(nodes), **kwargs) + for power_status_entry in power_status_all['status']: + # If the returned xname has an error, it itself is the status regardless of + # what the powerState field suggests. This is a major departure from how CAPMC + # handled errors. + xname = power_status_entry.get('xname', '') + if power_status_entry['error']: + status_bucket[power_status_entry['error']].add(xname) + continue + power_status = power_status_entry.get('powerState', '').lower() + if not all([power_status, xname]): + continue + status_bucket[power_status].add(xname) + return status_bucket + + def node_to_powerstate(self, nodes, **kwargs): + """ + For an iterable of nodes ; return a dictionary that maps to the current power state + for the node in question. + """ + power_states = {} + if not nodes: + LOGGER.warning( + "node_to_powerstate called without nodes; returning without action." + ) + return power_states + status_bucket = self.status(nodes, **kwargs) + for pstatus, nodeset in status_bucket.items(): + for node in nodeset: + power_states[node] = pstatus + return power_states diff --git a/src/bos/common/clients/pcs/transitions.py b/src/bos/common/clients/pcs/transitions.py new file mode 100644 index 00000000..c54ebe0e --- /dev/null +++ b/src/bos/common/clients/pcs/transitions.py @@ -0,0 +1,149 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +import logging + +from .base import BasePcsEndpoint +from .exceptions import PowerControlComponentsEmptyException, PowerControlSyntaxException + +LOGGER = logging.getLogger(__name__) + + +class TransitionsEndpoint(BasePcsEndpoint): + ENDPOINT = 'transitions' + + def transition_create(self, + xnames, + operation, + task_deadline_minutes=None, + deputy_key=None): + """ + Interact with PCS to create a request to transition one or more xnames. The transition + operation indicates what the desired operation should be, which is a string value + containing one or more of the supported transition names for the given hardware, e.g. 'on', + 'off', or 'force-off'. + + Once created, one of two responses are returned. A 2XX response results in a + transition_start_output object, or, an invalid request results in a 4XX and subsequent + raised PCS exception. + + Args: + xnames: an iterable of xnames + operation: A string/enum for what the nodes should transition to + task_deadline_minutes: How long should PCS operate on the nodes to bring them to + complete the operation; typecast to an integer value. + deputy_key: An optional string value that can be used to further handle instructing PCS + to perform state transitions on behalf of a known existing reservation. + session: An already existing session to use with PCS, if any + + Returns: + A transition_start_output object, which is a record for the transition that was + created. The most important field of which is the 'transitionID' value, which allows + subsequent follow-on to the created request. + + Raises: + PowerControlException: Any non-nominal response from PCS, typically as a result of an + unexpected payload response, or a failure to create a transition record. + PowerControlComponentsEmptyException: No xnames specified + """ + if not xnames: + raise PowerControlComponentsEmptyException( + f"_transition_create called with no xnames! (operation={operation})" + ) + try: + assert operation in { + 'On', 'Off', 'Soft-Off', 'Soft-Restart', 'Hard-Restart', + 'Init', 'Force-Off' + } + except AssertionError as err: + raise PowerControlSyntaxException( + f"Operation '{operation}' is not supported or implemented." + ) from err + params = {'location': [], 'operation': operation} + if task_deadline_minutes: + params['taskDeadlineMinutes'] = int(task_deadline_minutes) + for xname in xnames: + reserved_location = {'xname': xname} + if deputy_key: + reserved_location['deputyKey'] = deputy_key + params['location'].append(reserved_location) + return self.post(json=params) + + def power_on(self, nodes, task_deadline_minutes=1, **kwargs): + """ + Sends a request to PCS for transitioning nodes in question to a powered on state. + Returns: A JSON parsed object response from PCS, which includes the created request ID. + """ + if not nodes: + raise PowerControlComponentsEmptyException( + "power_on called with no nodes!") + return self.transition_create( + xnames=nodes, + operation='On', + task_deadline_minutes=task_deadline_minutes, + **kwargs) + + def power_off(self, nodes, task_deadline_minutes=1, **kwargs): + """ + Sends a request to PCS for transitioning nodes in question to a powered off state + (graceful). + Returns: A JSON parsed object response from PCS, which includes the created request ID. + """ + if not nodes: + raise PowerControlComponentsEmptyException( + "power_off called with no nodes!") + return self.transition_create( + xnames=nodes, + operation='Off', + task_deadline_minutes=task_deadline_minutes, + **kwargs) + + def soft_off(self, nodes, task_deadline_minutes=1, **kwargs): + """ + Sends a request to PCS for transitioning nodes in question to a powered off state + (graceful). + Returns: A JSON parsed object response from PCS, which includes the created request ID. + """ + if not nodes: + raise PowerControlComponentsEmptyException( + "soft_off called with no nodes!") + return self.transition_create( + xnames=nodes, + operation='Soft-Off', + task_deadline_minutes=task_deadline_minutes, + **kwargs) + + def force_off(self, nodes, task_deadline_minutes=1, **kwargs): + """ + Sends a request to PCS for transitioning nodes in question to a powered off state + (forceful). + Returns: A JSON parsed object response from PCS, which includes the created request ID. + """ + if not nodes: + raise PowerControlComponentsEmptyException( + "force_off called with no nodes!") + return self.transition_create( + xnames=nodes, + operation='Force-Off', + task_deadline_minutes=task_deadline_minutes, + **kwargs) diff --git a/src/bos/operators/utils/clients/s3.py b/src/bos/common/clients/s3.py similarity index 91% rename from src/bos/operators/utils/clients/s3.py rename to src/bos/common/clients/s3.py index 4b9afb3b..b12c685f 100644 --- a/src/bos/operators/utils/clients/s3.py +++ b/src/bos/common/clients/s3.py @@ -33,12 +33,13 @@ from bos.common.utils import exc_type_msg -LOGGER = logging.getLogger('bos.operators.utils.clients.s3') +LOGGER = logging.getLogger(__name__) # CASMCMS-9015: Instantiating the client is not thread-safe. # This lock is used to serialize it. boto3_client_lock = threading.Lock() + class ArtifactNotFound(Exception): """ A boot artifact could not be located. @@ -124,9 +125,8 @@ def s3_client(connection_timeout=60, read_timeout=60): aws_secret_access_key=s3_secret_key, use_ssl=False, verify=False, - config=BotoConfig( - connect_timeout=connection_timeout, - read_timeout=read_timeout)) + config=BotoConfig(connect_timeout=connection_timeout, + read_timeout=read_timeout)) return s3 @@ -160,10 +160,8 @@ def object_header(self) -> dict: try: s3 = s3_client() - s3_obj = s3.head_object( - Bucket=self.s3url.bucket, - Key=self.s3url.key - ) + s3_obj = s3.head_object(Bucket=self.s3url.bucket, + Key=self.s3url.key) except ClientError as error: msg = f"s3 object {self.path} was not found." LOGGER.error(msg) @@ -171,9 +169,10 @@ def object_header(self) -> dict: raise S3ObjectNotFound(msg) from error if self.etag and self.etag != s3_obj["ETag"].strip('\"'): - LOGGER.warning("s3 object %s was found, but has an etag '%s' that does " - "not match what BOS has '%s'.", self.path, s3_obj["ETag"], - self.etag) + LOGGER.warning( + "s3 object %s was found, but has an etag '%s' that does " + "not match what BOS has '%s'.", self.path, s3_obj["ETag"], + self.etag) return s3_obj @property @@ -194,7 +193,8 @@ def object(self): s3 = s3_client() - LOGGER.info("++ _get_s3_download_url %s with etag %s.", self.path, self.etag) + LOGGER.info("++ _get_s3_download_url %s with etag %s.", self.path, + self.etag) try: return s3.get_object(Bucket=self.s3url.bucket, Key=self.s3url.key) except (ClientError, ParamValidationError) as error: @@ -272,8 +272,10 @@ def _get_artifact(self, artifact_type): TooManyArtifacts -- There is more than one artifact when only one was expected """ try: - artifacts = [artifact for artifact in self.manifest_json['artifacts'] if - artifact['type'] == artifact_type] + artifacts = [ + artifact for artifact in self.manifest_json['artifacts'] + if artifact['type'] == artifact_type + ] except ValueError as value_error: LOGGER.info("Received ValueError while processing manifest file.") LOGGER.debug(value_error) @@ -317,7 +319,8 @@ def boot_parameters(self): boot parameters object if one exists, else None """ try: - bp = self._get_artifact('application/vnd.cray.image.parameters.boot') + bp = self._get_artifact( + 'application/vnd.cray.image.parameters.boot') except ArtifactNotFound: bp = None diff --git a/src/bos/common/tenant_utils.py b/src/bos/common/tenant_utils.py index 99b4d5b3..d41f16fc 100644 --- a/src/bos/common/tenant_utils.py +++ b/src/bos/common/tenant_utils.py @@ -25,19 +25,21 @@ import functools import logging import hashlib +from typing import Optional import connexion +import requests from requests.exceptions import HTTPError -from bos.common.utils import exc_type_msg, requests_retry_session, PROTOCOL +from bos.common.utils import exc_type_msg, retry_session_get, PROTOCOL -LOGGER = logging.getLogger('bos.common.tenant_utils') +LOGGER = logging.getLogger(__name__) TENANT_HEADER = "Cray-Tenant-Name" -SERVICE_NAME = 'cray-tapms/v1alpha3' # CASMCMS-9125: Currently when TAPMS bumps this version, it - # breaks backwards compatiblity, so BOS needs to update this - # whenever TAPMS does. +SERVICE_NAME = 'cray-tapms/v1alpha3' # CASMCMS-9125: Currently when TAPMS bumps this version, it +# breaks backwards compatiblity, so BOS needs to update this +# whenever TAPMS does. BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}" -TENANT_ENDPOINT = f"{BASE_ENDPOINT}/tenants" # CASMPET-6433 changed this from tenant to tenants +TENANT_ENDPOINT = f"{BASE_ENDPOINT}/tenants" # CASMPET-6433 changed this from tenant to tenants class InvalidTenantException(Exception): @@ -73,19 +75,19 @@ def get_tenant_aware_key(key, tenant): return f"{tenant_hash}-{key_hash}" -def get_tenant_data(tenant, session=None): - if not session: - session = requests_retry_session() +def get_tenant_data(tenant, session: Optional[requests.Session] = None): url = f"{TENANT_ENDPOINT}/{tenant}" - response = session.get(url) - try: - response.raise_for_status() - except HTTPError as e: - LOGGER.error("Failed getting tenant data from tapms: %s", exc_type_msg(e)) - if response.status_code == 404: - raise InvalidTenantException(f"Data not found for tenant {tenant}") from e - raise - return response.json() + with retry_session_get(url, session=session) as response: + try: + response.raise_for_status() + except HTTPError as e: + LOGGER.error("Failed getting tenant data from tapms: %s", + exc_type_msg(e)) + if response.status_code == 404: + raise InvalidTenantException( + f"Data not found for tenant {tenant}") from e + raise + return response.json() def get_tenant_component_set(tenant: str) -> set: @@ -93,7 +95,7 @@ def get_tenant_component_set(tenant: str) -> set: data = get_tenant_data(tenant) status = data.get("status", {}) for resource in status.get("tenantresources", []): - components.append(resource.get("xnames",[])) + components.append(resource.get("xnames", [])) return set().union(*components) @@ -107,27 +109,32 @@ def validate_tenant_exists(tenant: str) -> bool: def tenant_error_handler(func): """Decorator for returning errors if there is an exception when calling tapms""" + @functools.wraps(func) def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except InvalidTenantException as e: LOGGER.debug("Invalid tenant: %s", exc_type_msg(e)) - return connexion.problem( - status=400, title='Invalid tenant', - detail=str(e)) + return connexion.problem(status=400, + title='Invalid tenant', + detail=str(e)) + return wrapper def reject_invalid_tenant(func): """Decorator for preemptively validating the tenant exists""" + @functools.wraps(func) def wrapper(*args, **kwargs): tenant = get_tenant_from_header() if tenant and not validate_tenant_exists(tenant): LOGGER.debug("The provided tenant does not exist") return connexion.problem( - status=400, title="Invalid tenant", + status=400, + title="Invalid tenant", detail=str("The provided tenant does not exist")) return func(*args, **kwargs) + return wrapper diff --git a/src/bos/common/utils.py b/src/bos/common/utils.py index 35fe4df5..331d8f5f 100644 --- a/src/bos/common/utils.py +++ b/src/bos/common/utils.py @@ -23,18 +23,21 @@ # # Standard imports +from contextlib import nullcontext import datetime from functools import partial import re import traceback -from typing import List +from typing import Iterator, Optional, Unpack # Third party imports from dateutil.parser import parse -from requests_retry_session import requests_retry_session as base_requests_retry_session +import requests +import requests_retry_session as rrs PROTOCOL = 'http' -TIME_DURATION_PATTERN = re.compile(r"^(\d+?)(\D+?)$", re.M|re.S) +TIME_DURATION_PATTERN = re.compile(r"^(\d+?)(\D+?)$", re.M | re.S) + # Common date and timestamps functions so that timezones and formats are handled consistently. def get_current_time() -> datetime.datetime: @@ -54,21 +57,69 @@ def duration_to_timedelta(timestamp: str): Converts a to a timedelta object. """ # Calculate the corresponding multiplier for each time value - seconds_table = {'s': 1, - 'm': 60, - 'h': 60*60, - 'd': 60*60*24, - 'w': 60*60*24*7} + seconds_table = { + 's': 1, + 'm': 60, + 'h': 60 * 60, + 'd': 60 * 60 * 24, + 'w': 60 * 60 * 24 * 7 + } timeval, durationval = TIME_DURATION_PATTERN.search(timestamp).groups() timeval = float(timeval) seconds = timeval * seconds_table[durationval] return datetime.timedelta(seconds=seconds) -requests_retry_session = partial(base_requests_retry_session, - retries=10, backoff_factor=0.5, - status_forcelist=(500, 502, 503, 504), - connect_timeout=3, read_timeout=10, - session=None, protocol=PROTOCOL) + +DEFAULT_RETRY_ADAPTER_ARGS = rrs.RequestsRetryAdapterArgs( + retries=10, + backoff_factor=0.5, + status_forcelist=(500, 502, 503, 504), + connect_timeout=3, + read_timeout=10) + +retry_session_manager = partial(rrs.retry_session_manager, + protocol=PROTOCOL, + **DEFAULT_RETRY_ADAPTER_ARGS) + + +class RetrySessionManager(rrs.RetrySessionManager): + """ + Just sets the default values we use for our requests sessions + """ + + def __init__(self, + protocol: str = PROTOCOL, + **adapter_kwargs: Unpack[rrs.RequestsRetryAdapterArgs]): + for key, value in DEFAULT_RETRY_ADAPTER_ARGS.items(): + if key not in adapter_kwargs: + adapter_kwargs[key] = value + super().__init__(protocol=protocol, **adapter_kwargs) + + +def retry_session( + session: Optional[requests.Session] = None, + protocol: Optional[str] = None, + adapter_kwargs: Optional[rrs.RequestsRetryAdapterArgs] = None +) -> Iterator[requests.Session]: + if session is not None: + return nullcontext(session) + kwargs = adapter_kwargs or {} + if protocol is not None: + return retry_session_manager(protocol=protocol, **kwargs) # pylint: disable=redundant-keyword-arg + return retry_session_manager(**kwargs) + + +def retry_session_get(*get_args, + session: Optional[requests.Session] = None, + protocol: Optional[str] = None, + adapter_kwargs: Optional[ + rrs.RequestsRetryAdapterArgs] = None, + **get_kwargs) -> Iterator[requests.Response]: + with retry_session(session=session, + protocol=protocol, + adapter_kwargs=adapter_kwargs) as _session: + return _session.get(*get_args, **get_kwargs) + def compact_response_text(response_text: str) -> str: """ @@ -77,7 +128,7 @@ def compact_response_text(response_text: str) -> str: trailing whitespace from each line, and then returns it. """ if response_text: - return ' '.join([ line.strip() for line in response_text.split('\n') ]) + return ' '.join([line.strip() for line in response_text.split('\n')]) return str(response_text) @@ -88,6 +139,7 @@ def exc_type_msg(exc: Exception) -> str: """ return ''.join(traceback.format_exception_only(type(exc), exc)) + def get_image_id(component: str) -> str: """ Extract the IMS image ID from the path to the kernel @@ -95,7 +147,8 @@ def get_image_id(component: str) -> str: s3://boot-images/fbcc5b02-b6a4-46a8-9402-2b7138adc327/kernel """ # Get kernel's path - boot_artifacts = component.get('desired_state', {}).get('boot_artifacts', {}) + boot_artifacts = component.get('desired_state', + {}).get('boot_artifacts', {}) kernel = boot_artifacts.get('kernel') image_id = get_image_id_from_kernel(kernel) return image_id @@ -108,6 +161,7 @@ def get_image_id_from_kernel(kernel_path: str) -> str: image_id = match.group(1) return image_id + def using_sbps(component: str) -> bool: """ If the component is using the Scalable Boot Provisioning Service (SBPS) to @@ -120,10 +174,12 @@ def using_sbps(component: str) -> bool: Return True if it is and False if it is not. """ # Get the kernel boot parameters - boot_artifacts = component.get('desired_state', {}).get('boot_artifacts', {}) + boot_artifacts = component.get('desired_state', + {}).get('boot_artifacts', {}) kernel_parameters = boot_artifacts.get('kernel_parameters') return using_sbps_check_kernel_parameters(kernel_parameters) + def using_sbps_check_kernel_parameters(kernel_parameters: str) -> bool: """ Check the kernel boot parameters to see if the image is using the @@ -137,7 +193,8 @@ def using_sbps_check_kernel_parameters(kernel_parameters: str) -> bool: # Check for the 'root=sbps-s3' string. return "root=sbps-s3" in kernel_parameters -def components_by_id(components: List[dict]) -> dict: + +def components_by_id(components: list[dict]) -> dict: """ Input: * components: a list containing individual components @@ -148,9 +205,10 @@ def components_by_id(components: List[dict]) -> dict: Purpose: It makes searching more efficient because you can index by component name. """ - return { component["id"]: component for component in components } + return {component["id"]: component for component in components} + -def reverse_components_by_id(components_by_id_map: dict) -> List[dict]: +def reverse_components_by_id(components_by_id_map: dict) -> list[dict]: """ Input: components_by_id_map: a dictionary with the name of each component as the diff --git a/src/bos/operators/actual_state_cleanup.py b/src/bos/operators/actual_state_cleanup.py index b6c92ba2..445008cd 100644 --- a/src/bos/operators/actual_state_cleanup.py +++ b/src/bos/operators/actual_state_cleanup.py @@ -26,11 +26,11 @@ from bos.common.utils import duration_to_timedelta from bos.common.values import EMPTY_ACTUAL_STATE -from bos.operators.utils.clients.bos.options import options +from bos.common.clients.bos.options import options from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, ActualStateAge, ActualBootStateIsSet +from bos.operators.filters import ActualStateAge, ActualBootStateIsSet -LOGGER = logging.getLogger('bos.operators.actual_state_cleanup') +LOGGER = logging.getLogger(__name__) class ActualStateCleanupOperator(BaseOperator): @@ -53,22 +53,23 @@ def name(self): @property def filters(self): return [ - BOSQuery(), + self.BOSQuery(), ActualBootStateIsSet(), - ActualStateAge( - seconds=duration_to_timedelta(options.component_actual_state_ttl).total_seconds() - ) + ActualStateAge(seconds=duration_to_timedelta( + options.component_actual_state_ttl).total_seconds()) ] def _act(self, components): data = [] for component_id in [component['id'] for component in components]: - data.append({'id': component_id, - 'actual_state': EMPTY_ACTUAL_STATE}) + data.append({ + 'id': component_id, + 'actual_state': EMPTY_ACTUAL_STATE + }) if data: LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Calling to update with payload: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) return components diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 1f4e0c74..8230db09 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -27,6 +27,7 @@ """ from abc import ABC, abstractmethod +from contextlib import ExitStack import itertools import logging import threading @@ -36,12 +37,18 @@ from bos.common.utils import exc_type_msg from bos.common.values import Status +from bos.operators.filters import BOSQuery, DesiredConfigurationSetInCFS, HSMState from bos.operators.filters.base import BaseFilter -from bos.operators.utils.clients.bos.options import options -from bos.operators.utils.clients.bos import BOSClient +from bos.common.clients.bos.options import options +from bos.common.clients.bos import BOSClient +from bos.common.clients.bss import BSSClient +from bos.common.clients.cfs import CFSClient +from bos.common.clients.hsm import HSMClient +from bos.common.clients.ims import IMSClient +from bos.common.clients.pcs import PCSClient from bos.operators.utils.liveness.timestamp import Timestamp -LOGGER = logging.getLogger('bos.operators.base') +LOGGER = logging.getLogger(__name__) MAIN_THREAD = threading.current_thread() @@ -56,6 +63,30 @@ class MissingSessionData(BaseOperatorException): """ +class ApiClients: + + def __init__(self): + self.bos = BOSClient() + self.bss = BSSClient() + self.cfs = CFSClient() + self.hsm = HSMClient() + self.ims = IMSClient() + self.pcs = PCSClient() + self._stack = ExitStack() + + def __enter__(self): + self._stack.enter_context(self.bos) + self._stack.enter_context(self.bss) + self._stack.enter_context(self.cfs) + self._stack.enter_context(self.hsm) + self._stack.enter_context(self.ims) + self._stack.enter_context(self.pcs) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return self._stack.__exit__(exc_type, exc_val, exc_tb) + + class BaseOperator(ABC): """ An abstract class for all BOS operators. @@ -74,8 +105,14 @@ class BaseOperator(ABC): frequency_option = "polling_frequency" def __init__(self) -> NoReturn: - self.bos_client = BOSClient() self.__max_batch_size = 0 + self._client: ApiClients | None = None + + @property + def client(self) -> ApiClients: + if self._client is None: + raise ValueError("Attempted to access uninitialized API client") + return self._client @property @abstractmethod @@ -87,6 +124,28 @@ def name(self) -> str: def filters(self) -> List[Type[BaseFilter]]: return [] + def BOSQuery(self, **kwargs) -> BOSQuery: + """ + Shortcut to get a BOSQuery filter with the bos_client for this operator + """ + if 'bos_client' not in kwargs: + kwargs['bos_client'] = self.client.bos + return BOSQuery(**kwargs) + + def DesiredConfigurationSetInCFS(self) -> DesiredConfigurationSetInCFS: + """ + Shortcut to get a DesiredConfigurationSetInCFS filter with the cfs_client for this operator + """ + return DesiredConfigurationSetInCFS(self.client.cfs) + + def HSMState(self, **kwargs) -> HSMState: + """ + Shortcut to get a HSMState filter with the bos_client for this operator + """ + if 'hsm_client' not in kwargs: + kwargs['hsm_client'] = self.client.hsm + return HSMState(**kwargs) + def run(self) -> NoReturn: """ The core method of the operator that periodically detects and acts on components. @@ -98,27 +157,37 @@ def run(self) -> NoReturn: try: options.update() _update_log_level() - self._run() + with ApiClients() as _client: + self._client = _client + self._run() except Exception as e: LOGGER.exception('Unhandled exception detected: %s', e) + finally: + self._client = None try: - sleep_time = getattr(options, self.frequency_option) - (time.time() - start_time) + sleep_time = getattr(options, self.frequency_option) - ( + time.time() - start_time) if sleep_time > 0: time.sleep(sleep_time) except Exception as e: - LOGGER.exception('Unhandled exception getting polling frequency: %s', e) - time.sleep(5) # A small sleep for when exceptions getting the polling frequency + LOGGER.exception( + 'Unhandled exception getting polling frequency: %s', e) + time.sleep( + 5 + ) # A small sleep for when exceptions getting the polling frequency @property def max_batch_size(self) -> int: max_batch_size = options.max_component_batch_size if max_batch_size != self.__max_batch_size: - LOGGER.info("max_component_batch_size option set to %d", max_batch_size) + LOGGER.info("max_component_batch_size option set to %d", + max_batch_size) self.__max_batch_size = max_batch_size return max_batch_size - def _chunk_components(self, components: List[dict]) -> Generator[List[dict], None, None]: + def _chunk_components( + self, components: List[dict]) -> Generator[List[dict], None, None]: """ Break up the components into groups of no more than max_batch_size nodes, and yield each group in turn. @@ -145,16 +214,19 @@ def _run_on_chunk(self, components: List[dict]) -> None: if self.retry_attempt_field: components = self._handle_failed_components(components) if not components: - LOGGER.debug('After removing components that exceeded their retry limit, 0 ' - 'components require action') + LOGGER.debug( + 'After removing components that exceeded their retry limit, 0 ' + 'components require action') return for component in components: # Unset old errors components component['error'] = '' try: components = self._act(components) except Exception as e: - LOGGER.error("An unhandled exception was caught while trying to act on components: %s", - e, exc_info=True) + LOGGER.error( + "An unhandled exception was caught while trying to act on components: %s", + e, + exc_info=True) for component in components: component["error"] = str(e) self._update_database(components) @@ -173,10 +245,13 @@ def _handle_failed_components(self, components: List[dict]) -> List[dict]: LOGGER.debug("_handle_failed_components: No components to handle") return [] failed_components = [] - good_components = [] # Any component that isn't determined to be in a failed state + good_components = [ + ] # Any component that isn't determined to be in a failed state for component in components: - num_attempts = component.get('event_stats', {}).get(self.retry_attempt_field, 0) - retries = int(component.get('retry_policy', options.default_retry_policy)) + num_attempts = component.get('event_stats', + {}).get(self.retry_attempt_field, 0) + retries = int( + component.get('retry_policy', options.default_retry_policy)) if retries != -1 and num_attempts >= retries: # This component has hit its retry limit failed_components.append(component) @@ -190,35 +265,35 @@ def _act(self, components: List[dict]) -> List[dict]: """ The action taken by the operator on target components """ raise NotImplementedError() - def _update_database(self, components: List[dict], additional_fields: dict=None) -> None: + def _update_database(self, + components: List[dict], + additional_fields: dict = None) -> None: """ Updates the BOS database for all components acted on by the operator Includes updating the last action, attempt count and error """ if not components: # If we have been passed an empty list, there is nothing to do. - LOGGER.debug("_update_database: No components require database updates") + LOGGER.debug( + "_update_database: No components require database updates") return data = [] for component in components: patch = { 'id': component['id'], - 'error': component['error'] # New error, or clearing out old error + 'error': + component['error'] # New error, or clearing out old error } if self.name: - last_action_data = { - 'action': self.name, - 'failed': False - } + last_action_data = {'action': self.name, 'failed': False} patch['last_action'] = last_action_data if self.retry_attempt_field: event_stats_data = { - self.retry_attempt_field: component.get( - 'event_stats', - {} - ).get(self.retry_attempt_field, 0) + 1 + self.retry_attempt_field: + component.get('event_stats', {}).get( + self.retry_attempt_field, 0) + 1 } - patch['event_stats'] = event_stats_data + patch['event_stats'] = event_stats_data if additional_fields: patch.update(additional_fields) @@ -232,7 +307,7 @@ def _update_database(self, components: List[dict], additional_fields: dict=None) data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) def _preset_last_action(self, components: List[dict]) -> None: # This is done to eliminate the window between performing an action and marking the @@ -243,24 +318,19 @@ def _preset_last_action(self, components: List[dict]) -> None: return if not components: # If we have been passed an empty list, there is nothing to do. - LOGGER.debug("_preset_last_action: No components require database updates") + LOGGER.debug( + "_preset_last_action: No components require database updates") return data = [] for component in components: - patch = { - 'id': component['id'], - 'error': component['error'] - } + patch = {'id': component['id'], 'error': component['error']} if self.name: - last_action_data = { - 'action': self.name, - 'failed': False - } + last_action_data = {'action': self.name, 'failed': False} patch['last_action'] = last_action_data data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) def _update_database_for_failure(self, components: List[dict]) -> None: """ @@ -268,21 +338,26 @@ def _update_database_for_failure(self, components: List[dict]) -> None: """ if not components: # If we have been passed an empty list, there is nothing to do. - LOGGER.debug("_update_database_for_failure: No components require database updates") + LOGGER.debug( + "_update_database_for_failure: No components require database updates" + ) return data = [] for component in components: patch = { 'id': component['id'], - 'status': {'status_override': Status.failed} + 'status': { + 'status_override': Status.failed + } } if not component['error']: - patch['error'] = ('The retry limit has been hit for this component, ' - 'but no services have reported specific errors') + patch['error'] = ( + 'The retry limit has been hit for this component, ' + 'but no services have reported specific errors') data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) def chunk_components(components: List[dict], @@ -336,7 +411,8 @@ def _init_logging() -> None: log_level = logging.getLevelName(requested_log_level) if not isinstance(log_level, int): - LOGGER.warning('Log level %r is not valid. Falling back to INFO', requested_log_level) + LOGGER.warning('Log level %r is not valid. Falling back to INFO', + requested_log_level) log_level = logging.INFO logging.basicConfig(level=log_level, format=log_format) diff --git a/src/bos/operators/configuration.py b/src/bos/operators/configuration.py index fa6f1975..3d417341 100644 --- a/src/bos/operators/configuration.py +++ b/src/bos/operators/configuration.py @@ -25,11 +25,10 @@ import logging from bos.common.values import Status -from bos.operators.utils.clients.cfs import set_cfs from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, DesiredConfigurationSetInCFS, NOT +from bos.operators.filters import NOT -LOGGER = logging.getLogger('bos.operators.configuration') +LOGGER = logging.getLogger(__name__) class ConfigurationOperator(BaseOperator): @@ -50,13 +49,13 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=Status.configuring), - NOT(DesiredConfigurationSetInCFS()) + self.BOSQuery(enabled=True, status=Status.configuring), + NOT(self.DesiredConfigurationSetInCFS()) ] def _act(self, components): if components: - set_cfs(components, enabled=True) + self.client.cfs.components.set_cfs(components, enabled=True) return components diff --git a/src/bos/operators/discovery.py b/src/bos/operators/discovery.py index a7110ae5..02dd4ab7 100644 --- a/src/bos/operators/discovery.py +++ b/src/bos/operators/discovery.py @@ -27,19 +27,23 @@ from copy import copy from bos.common.values import Action, EMPTY_ACTUAL_STATE, EMPTY_DESIRED_STATE -from bos.operators.utils.clients.hsm import read_all_node_xnames from bos.operators.base import BaseOperator, main LOGGER = logging.getLogger(__name__) -NEW_COMPONENT = {'id': None, - 'actual_state': EMPTY_ACTUAL_STATE, - 'desired_state': EMPTY_DESIRED_STATE, - 'staged_state': {}, - 'last_action': {'action': Action.newly_discovered}, - 'enabled': False, - 'error': '', - 'session': ''} +NEW_COMPONENT = { + 'id': None, + 'actual_state': EMPTY_ACTUAL_STATE, + 'desired_state': EMPTY_DESIRED_STATE, + 'staged_state': {}, + 'last_action': { + 'action': Action.newly_discovered + }, + 'enabled': False, + 'error': '', + 'session': '' +} + class DiscoveryOperator(BaseOperator): """ @@ -79,7 +83,7 @@ def _run(self) -> None: return LOGGER.info("%s new component(s) from HSM.", len(components_to_add)) for chunk in self._chunk_components(components_to_add): - self.bos_client.components.put_components(chunk) + self.client.bos.components.put_components(chunk) LOGGER.info("%s new component(s) added to BOS!", len(chunk)) @property @@ -88,7 +92,7 @@ def bos_components(self) -> Set[str]: The set of components currently known to BOS """ components = set() - for component in self.bos_client.components.get_components(): + for component in self.client.bos.components.get_components(): components.add(component['id']) return components @@ -97,7 +101,7 @@ def hsm_xnames(self) -> Set[str]: """ The set of components currently known to HSM State Manager """ - return read_all_node_xnames() + return self.client.hsm.state_components.read_all_node_xnames() @property def missing_components(self) -> Set[str]: @@ -106,5 +110,6 @@ def missing_components(self) -> Set[str]: """ return self.hsm_xnames - self.bos_components + if __name__ == '__main__': main(DiscoveryOperator) diff --git a/src/bos/operators/filters/base.py b/src/bos/operators/filters/base.py index 5f03c236..3846e688 100644 --- a/src/bos/operators/filters/base.py +++ b/src/bos/operators/filters/base.py @@ -26,8 +26,7 @@ import logging from typing import List - -LOGGER = logging.getLogger('bos.operators.filters.base') +LOGGER = logging.getLogger(__name__) # Abstracts @@ -57,20 +56,26 @@ def _filter(self, components: List) -> List: class IDFilter(BaseFilter, ABC): """ A class for filters that take and return lists of component ids """ + def filter(self, components: List[dict]) -> List[dict]: component_ids = [component['id'] for component in components] results = BaseFilter.filter(self, components=component_ids) - LOGGER.debug('%s filter found the following components: %s', type(self).__name__, - ','.join(results)) - return [component for component in components if component['id'] in results] + LOGGER.debug('%s filter found the following components: %s', + type(self).__name__, ','.join(results)) + return [ + component for component in components if component['id'] in results + ] class DetailsFilter(BaseFilter, ABC): """ A class for filters that take and return lists of detailed component information """ + def filter(self, components: List[dict]) -> List[dict]: results = BaseFilter.filter(self, components=components) - LOGGER.debug('%s filter found the following components: %s', type(self).__name__, - ','.join([component.get('id', '') for component in results])) + LOGGER.debug( + '%s filter found the following components: %s', + type(self).__name__, + ','.join([component.get('id', '') for component in results])) return results @@ -79,6 +84,7 @@ class LocalFilter(DetailsFilter, ABC): A class for filters that loop over component information that is already obtained. Only the _match method needs to be overridden to filter on one component at a time. """ + def _filter(self, components: List[dict]) -> List[dict]: matching_components = [] for component in components: diff --git a/src/bos/operators/filters/filters.py b/src/bos/operators/filters/filters.py index df51314d..80d8d3d9 100644 --- a/src/bos/operators/filters/filters.py +++ b/src/bos/operators/filters/filters.py @@ -30,12 +30,11 @@ from bos.common.utils import get_current_time, load_timestamp from bos.operators.filters.base import BaseFilter, DetailsFilter, IDFilter, LocalFilter -from bos.operators.utils.clients.bos import BOSClient -from bos.operators.utils.clients.cfs import get_components_from_id_list as \ - get_cfs_components_from_id_list -from bos.operators.utils.clients.hsm import get_components as get_hsm_components +from bos.common.clients.bos import BOSClient +from bos.common.clients.cfs import CFSClient +from bos.common.clients.hsm import HSMClient -LOGGER = logging.getLogger('bos.operators.filters.filters') +LOGGER = logging.getLogger(__name__) # Usable filters @@ -53,8 +52,14 @@ def _filter(self, components: List[dict]) -> List[dict]: results_b = copy.deepcopy(components) for f in self.filters_b: results_b = f.filter(results_b) - results_a_dict = {component['id']: component for component in results_a} - results_b_dict = {component['id']: component for component in results_b} + results_a_dict = { + component['id']: component + for component in results_a + } + results_b_dict = { + component['id']: component + for component in results_b + } results = {**results_a_dict, **results_b_dict} return list(results.values()) @@ -63,14 +68,14 @@ class BOSQuery(DetailsFilter): """Gets all components from BOS that match the kwargs """ INITIAL: bool = True - def __init__(self, **kwargs) -> None: + def __init__(self, bos_client: BOSClient, **kwargs) -> None: """ Init for the BOSQuery filter kwargs corresponds to arguments for the BOS get_components method """ super().__init__() self.kwargs = kwargs - self.bos_client = BOSClient() + self.bos_client = bos_client def _filter(self, _) -> List[dict]: return self.bos_client.components.get_components(**self.kwargs) @@ -79,16 +84,23 @@ def _filter(self, _) -> List[dict]: class HSMState(IDFilter): """ Returns all components that are in specified state """ - def __init__(self, enabled: bool=None, ready: bool=None) -> None: + def __init__(self, + hsm_client: HSMClient, + enabled: bool = None, + ready: bool = None) -> None: super().__init__() self.enabled = enabled self.ready = ready + self.hsm_client = hsm_client def _filter(self, components: List[str]) -> List[str]: - components = get_hsm_components(components, enabled=self.enabled) + components = self.hsm_client.state_components.get_components( + components, enabled=self.enabled) if self.ready is not None: - return [component['ID'] for component in components['Components'] - if (component['State'] == 'Ready') is self.ready] + return [ + component['ID'] for component in components['Components'] + if (component['State'] == 'Ready') is self.ready + ] return [component['ID'] for component in components['Components']] def filter_by_arch(self, nodes, arch): @@ -103,9 +115,13 @@ def filter_by_arch(self, nodes, arch): returns: A list of xnames all matching one of the archs requested """ - components = get_hsm_components(list(nodes), enabled=self.enabled) - return [component['ID'] for component in components['Components'] - if component.get('Arch', 'Unknown') in arch] + components = self.hsm_client.state_components.get_components( + list(nodes), enabled=self.enabled) + return [ + component['ID'] for component in components['Components'] + if component.get('Arch', 'Unknown') in arch + ] + class NOT(LocalFilter): """ Returns the opposite of the given filter. Use on local filters only.""" @@ -113,8 +129,10 @@ class NOT(LocalFilter): def __init__(self, filter: Type[LocalFilter]) -> None: self.negated_filter = filter self.filter_match = filter._match + def negated_match(*args, **kwargs): return not self.filter_match(*args, **kwargs) + self.negated_filter._match = negated_match def _filter(self, components: List[dict]) -> List[dict]: @@ -168,13 +186,14 @@ def _match(self, component: dict) -> bool: desired_boot_state = desired_state.get('boot_artifacts', {}) actual_boot_state = actual_state.get('boot_artifacts', {}) for key in ['kernel', 'initrd']: - if desired_boot_state.get(key, None) != actual_boot_state.get(key, None): + if desired_boot_state.get(key, None) != actual_boot_state.get( + key, None): return False # Filter out kernel parameters that dynamically change. actual_kernel_parameters = self._sanitize_kernel_parameters( - actual_boot_state.get('kernel_parameters', None)) + actual_boot_state.get('kernel_parameters', None)) desired_kernel_parameters = self._sanitize_kernel_parameters( - desired_boot_state.get('kernel_parameters', None)) + desired_boot_state.get('kernel_parameters', None)) if actual_kernel_parameters != desired_kernel_parameters: return False @@ -193,26 +212,31 @@ def _sanitize_kernel_parameters(self, parameter_string): if not parameter_string: return None - return re.sub(r'(^\\s)+spire_join_token=[\S]*' , '', parameter_string) + return re.sub(r'(^\\s)+spire_join_token=[\S]*', '', parameter_string) class DesiredConfigurationSetInCFS(LocalFilter): """ Returns when desired configuration is set in CFS """ - def __init__(self): - self.cfs_components_dict = {} + def __init__(self, cfs_client: CFSClient): super().__init__() + self.cfs_components_dict = {} + self.cfs_client = cfs_client def _filter(self, components: List[dict]) -> List[dict]: component_ids = [component['id'] for component in components] - cfs_components = get_cfs_components_from_id_list(id_list=component_ids) - self.cfs_components_dict = {component['id']: component for component in cfs_components} + cfs_components = self.cfs_client.components.get_components_from_id_list( + id_list=component_ids) + self.cfs_components_dict = { + component['id']: component + for component in cfs_components + } matches = LocalFilter._filter(self, components) # Clear this, so there are no lingering side-effects of running this method. self.cfs_components_dict = {} return matches - def _match(self, component: dict, cfs_component: dict=None) -> bool: + def _match(self, component: dict, cfs_component: dict = None) -> bool: # There are two ways to communicate the cfs_component to this method. # First: cfs_component input variable # Second: cfs_component_dict instance attribute @@ -224,9 +248,11 @@ def _match(self, component: dict, cfs_component: dict=None) -> bool: # However, the status operator needs to pass in the cfs_component parameter # (i.e. the first method) because it is not calling the _filter method # which sets/updates the cfs_components_dict attribute. - desired_configuration = component.get('desired_state', {}).get('configuration') + desired_configuration = component.get('desired_state', + {}).get('configuration') if cfs_component is None: - set_configuration = self.cfs_components_dict[component['id']].get('desired_config') + set_configuration = self.cfs_components_dict[component['id']].get( + 'desired_config') else: set_configuration = cfs_component.get('desired_config') return desired_configuration == set_configuration @@ -238,7 +264,8 @@ class DesiredBootStateIsNone(LocalFilter): def _match(self, component: dict) -> bool: desired_state = component.get('desired_state', {}) desired_boot_state = desired_state.get('boot_artifacts', {}) - if not desired_boot_state or not any(bool(v) for v in desired_boot_state.values()): + if not desired_boot_state or not any( + bool(v) for v in desired_boot_state.values()): return True return False @@ -276,9 +303,11 @@ def __init__(self, **kwargs) -> None: self.kwargs = kwargs def _match(self, component: dict) -> bool: - last_updated = component.get('actual_state', {}).get('last_updated', None) + last_updated = component.get('actual_state', + {}).get('last_updated', None) now = get_current_time() - if not last_updated or now > load_timestamp(last_updated) + timedelta(**self.kwargs): + if not last_updated or now > load_timestamp(last_updated) + timedelta( + **self.kwargs): return True return False @@ -287,7 +316,8 @@ class ActualBootStateIsSet(LocalFilter): """ Returns when the actual state h """ def _match(self, component: dict) -> bool: - actual_state_boot_artifacts = component.get('actual_state', {}).get('boot_artifacts', {}) + actual_state_boot_artifacts = component.get('actual_state', {}).get( + 'boot_artifacts', {}) # The timestamp field doesn't count as a set record we particularly care about if 'timestamp' in actual_state_boot_artifacts: del actual_state_boot_artifacts['timestamp'] diff --git a/src/bos/operators/power_off_forceful.py b/src/bos/operators/power_off_forceful.py index 5d258b7a..e8c56522 100644 --- a/src/bos/operators/power_off_forceful.py +++ b/src/bos/operators/power_off_forceful.py @@ -25,12 +25,11 @@ import logging from bos.common.values import Action, Status -from bos.operators.utils.clients import pcs -from bos.operators.utils.clients.bos.options import options +from bos.common.clients.bos.options import options from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, HSMState, TimeSinceLastAction +from bos.operators.filters import TimeSinceLastAction -LOGGER = logging.getLogger('bos.operators.power_off_forceful') +LOGGER = logging.getLogger(__name__) class ForcefulPowerOffOperator(BaseOperator): @@ -50,16 +49,19 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=','.join([Status.power_off_forcefully_called, - Status.power_off_gracefully_called])), + self.BOSQuery(enabled=True, + status=','.join([ + Status.power_off_forcefully_called, + Status.power_off_gracefully_called + ])), TimeSinceLastAction(seconds=options.max_power_off_wait_time), - HSMState(), + self.HSMState(), ] def _act(self, components): if components: component_ids = [component['id'] for component in components] - pcs.force_off(nodes=component_ids) + self.client.pcs.transitions.force_off(component_ids) return components diff --git a/src/bos/operators/power_off_graceful.py b/src/bos/operators/power_off_graceful.py index 9a925b07..6c5849df 100644 --- a/src/bos/operators/power_off_graceful.py +++ b/src/bos/operators/power_off_graceful.py @@ -25,11 +25,9 @@ import logging from bos.common.values import Action, Status -from bos.operators.utils.clients import pcs from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, HSMState -LOGGER = logging.getLogger('bos.operators.power_off_graceful') +LOGGER = logging.getLogger(__name__) class GracefulPowerOffOperator(BaseOperator): @@ -48,14 +46,14 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=Status.power_off_pending), - HSMState(), + self.BOSQuery(enabled=True, status=Status.power_off_pending), + self.HSMState(), ] def _act(self, components): if components: component_ids = [component['id'] for component in components] - pcs.soft_off(component_ids) + self.client.pcs.transitions.soft_off(component_ids) return components diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index 8b06f84a..ff083c10 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -35,15 +35,10 @@ from bos.common.utils import exc_type_msg, get_image_id_from_kernel, \ using_sbps_check_kernel_parameters, components_by_id from bos.common.values import Action, Status -from bos.operators.utils.clients import bss -from bos.operators.utils.clients import pcs -from bos.operators.utils.clients.ims import tag_image -from bos.operators.utils.clients.cfs import set_cfs from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, HSMState from bos.server.dbs.boot_artifacts import record_boot_artifacts -LOGGER = logging.getLogger('bos.operators.power_on') +LOGGER = logging.getLogger(__name__) class PowerOnOperator(BaseOperator): @@ -63,16 +58,17 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=Status.power_on_pending), - HSMState() + self.BOSQuery(enabled=True, status=Status.power_on_pending), + self.HSMState() ] - def _act(self, components: Union[List[dict],None]): + def _act(self, components: Union[List[dict], None]): if not components: return components self._preset_last_action(components) - boot_artifacts, sessions = self._sort_components_by_boot_artifacts(components) + boot_artifacts, sessions = self._sort_components_by_boot_artifacts( + components) try: self._tag_images(boot_artifacts, components) @@ -81,19 +77,25 @@ def _act(self, components: Union[List[dict],None]): try: self._set_bss(boot_artifacts, bos_sessions=sessions) except Exception as e: - raise Exception(f"Error encountered setting BSS information: {e}") from e + raise Exception( + f"Error encountered setting BSS information: {e}") from e try: - set_cfs(components, enabled=False, clear_state=True) + self.client.cfs.components.set_cfs(components, + enabled=False, + clear_state=True) except Exception as e: - raise Exception(f"Error encountered setting CFS information: {e}") from e + raise Exception( + f"Error encountered setting CFS information: {e}") from e component_ids = [component['id'] for component in components] try: - pcs.power_on(component_ids) + self.client.pcs.transitions.power_on(component_ids) except Exception as e: - raise Exception(f"Error encountered calling CAPMC to power on: {e}") from e + raise Exception( + f"Error encountered calling CAPMC to power on: {e}") from e return components - def _sort_components_by_boot_artifacts(self, components: List[dict]) -> tuple[Dict, Dict]: + def _sort_components_by_boot_artifacts( + self, components: List[dict]) -> tuple[Dict, Dict]: """ Create a two dictionaries. The first dictionary has keys with a unique combination of boot artifacts associated with @@ -116,7 +118,8 @@ def _sort_components_by_boot_artifacts(self, components: List[dict]) -> tuple[Di bos_sessions = {} for component in components: # Handle the boot artifacts - nodes_boot_artifacts = component.get('desired_state', {}).get('boot_artifacts', {}) + nodes_boot_artifacts = component.get('desired_state', + {}).get('boot_artifacts', {}) kernel = nodes_boot_artifacts.get('kernel') kernel_parameters = nodes_boot_artifacts.get('kernel_parameters') initrd = nodes_boot_artifacts.get('initrd') @@ -147,42 +150,50 @@ def _set_bss(self, boot_artifacts, bos_sessions, retries=5): for key, nodes in boot_artifacts.items(): kernel, kernel_parameters, initrd = key try: - resp = bss.set_bss(node_set=nodes, kernel_params=kernel_parameters, - kernel=kernel, initrd=initrd) - resp.raise_for_status() + token = self.client.bss.boot_parameters.set_bss( + node_set=nodes, + kernel_params=kernel_parameters, + kernel=kernel, + initrd=initrd) except HTTPError as err: - LOGGER.error("Failed to set BSS for boot artifacts: %s for nodes: %s. Error: %s", - key, nodes, exc_type_msg(err)) + LOGGER.error( + "Failed to set BSS for boot artifacts: %s for nodes: %s. Error: %s", + key, nodes, exc_type_msg(err)) else: - token = resp.headers['bss-referral-token'] attempts = 0 while attempts <= retries: try: - record_boot_artifacts(token, kernel, kernel_parameters, initrd) + record_boot_artifacts(token, kernel, kernel_parameters, + initrd) break except Exception as err: attempts += 1 - LOGGER.error("An error occurred attempting to record the BSS token: %s", - exc_type_msg(err)) + LOGGER.error( + "An error occurred attempting to record the BSS token: %s", + exc_type_msg(err)) if attempts > retries: raise LOGGER.info("Retrying to record the BSS token.") for node in nodes: - bss_tokens.append({"id": node, - "desired_state": {"bss_token": token}, - "session": bos_sessions[node]}) - LOGGER.info('Found %d components that require BSS token updates', len(bss_tokens)) + bss_tokens.append({ + "id": node, + "desired_state": { + "bss_token": token + }, + "session": bos_sessions[node] + }) + LOGGER.info('Found %d components that require BSS token updates', + len(bss_tokens)) if not bss_tokens: return - redacted_component_updates = [ - { "id": comp["id"], - "session": comp["session"] - } - for comp in bss_tokens ] + redacted_component_updates = [{ + "id": comp["id"], + "session": comp["session"] + } for comp in bss_tokens] LOGGER.debug('Updated components (minus desired_state data): %s', redacted_component_updates) - self.bos_client.components.update_components(bss_tokens) + self.client.bos.components.update_components(bss_tokens) def _tag_images(self, boot_artifacts: Dict[Tuple[str, str, str], Set[str]], components: List[dict]) -> None: @@ -227,7 +238,8 @@ def _tag_images(self, boot_artifacts: Dict[Tuple[str, str, str], Set[str]], my_components_by_id = components_by_id(components) for image in image_ids: try: - tag_image(image, "set", "sbps-project", "true") + self.client.ims.images.tag_image(image, "set", "sbps-project", + "true") except Exception as e: components_to_update = [] for node in image_id_to_nodes[image]: @@ -235,5 +247,6 @@ def _tag_images(self, boot_artifacts: Dict[Tuple[str, str, str], Set[str]], components_to_update.append(my_components_by_id[node]) self._update_database(components_to_update) + if __name__ == '__main__': main(PowerOnOperator) diff --git a/src/bos/operators/session_cleanup.py b/src/bos/operators/session_cleanup.py index bf26ca64..0cf189bf 100644 --- a/src/bos/operators/session_cleanup.py +++ b/src/bos/operators/session_cleanup.py @@ -26,9 +26,9 @@ import re from bos.operators.base import BaseOperator, main -from bos.operators.utils.clients.bos.options import options +from bos.common.clients.bos.options import options -LOGGER = logging.getLogger('bos.operators.session_cleanup') +LOGGER = logging.getLogger(__name__) class SessionCleanupOperator(BaseOperator): @@ -36,6 +36,7 @@ class SessionCleanupOperator(BaseOperator): The Session Completion Operator marks sessions complete when all components that are part of the session have been disabled. """ + @property def name(self): return 'SessionCleanup' @@ -45,7 +46,8 @@ def disabled(self): """ When users set the cleanup time to 0, no cleanup behavior is desired. """ - options_stripped = re.sub('[^0-9]', '', options.cleanup_completed_session_ttl) + options_stripped = re.sub('[^0-9]', '', + options.cleanup_completed_session_ttl) return not bool(int(options_stripped)) # This operator overrides _run and does not use "filters" or "_act", but they are defined here @@ -66,9 +68,12 @@ def _run(self) -> None: if self.disabled: return - self.bos_client.sessions.delete_sessions( - **{'status': 'complete', - 'min_age': options.cleanup_completed_session_ttl}) + self.client.bos.sessions.delete_sessions( + **{ + 'status': 'complete', + 'min_age': options.cleanup_completed_session_ttl + }) + if __name__ == '__main__': main(SessionCleanupOperator) diff --git a/src/bos/operators/session_completion.py b/src/bos/operators/session_completion.py index 56076b36..8bc61c58 100644 --- a/src/bos/operators/session_completion.py +++ b/src/bos/operators/session_completion.py @@ -27,7 +27,7 @@ from bos.common.utils import get_current_timestamp from bos.operators.base import BaseOperator, main -LOGGER = logging.getLogger('bos.operators.session_completion') +LOGGER = logging.getLogger(__name__) class SessionCompletionOperator(BaseOperator): @@ -55,23 +55,28 @@ def _run(self) -> None: for session in sessions: components = self._get_incomplete_components(session["name"]) if not components: - self._mark_session_complete(session["name"], session.get("tenant")) + self._mark_session_complete(session["name"], + session.get("tenant")) def _get_incomplete_sessions(self): - return self.bos_client.sessions.get_sessions(status = 'running') + return self.client.bos.sessions.get_sessions(status='running') def _get_incomplete_components(self, session_id): - components = self.bos_client.components.get_components(session = session_id, enabled = True) - components += self.bos_client.components.get_components(staged_session = session_id) + components = self.client.bos.components.get_components( + session=session_id, enabled=True) + components += self.client.bos.components.get_components( + staged_session=session_id) return components def _mark_session_complete(self, session_id, tenant): - self.bos_client.sessions.update_session(session_id, tenant, - { 'status': { 'status': 'complete', - 'end_time': get_current_timestamp() - }}) + self.client.bos.sessions.update_session(session_id, tenant, { + 'status': { + 'status': 'complete', + 'end_time': get_current_timestamp() + } + }) # This call causes the session status to saved in the database. - self.bos_client.session_status.post_session_status(session_id, tenant) + self.client.bos.sessions.post_session_status(session_id, tenant) LOGGER.info('Session %s is complete', session_id) diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 6c7270fd..5d885547 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -28,18 +28,17 @@ from botocore.exceptions import ClientError from bos.operators.base import BaseOperator, main, chunk_components -from bos.operators.filters.filters import HSMState -from bos.operators.utils.clients.hsm import Inventory -from bos.operators.utils.clients.s3 import S3Object, S3ObjectNotFound +from bos.common.clients.hsm import Inventory +from bos.common.clients.s3 import S3Object, S3ObjectNotFound from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory -from bos.operators.utils.clients.bos.options import options +from bos.common.clients.bos.options import options from bos.operators.utils.rootfs.factory import ProviderFactory from bos.operators.session_completion import SessionCompletionOperator from bos.common.utils import exc_type_msg from bos.common.values import Action, EMPTY_ACTUAL_STATE, EMPTY_DESIRED_STATE, EMPTY_STAGED_STATE from bos.common.tenant_utils import get_tenant_component_set, InvalidTenantException -LOGGER = logging.getLogger('bos.operators.session_setup') +LOGGER = logging.getLogger(__name__) class SessionSetupException(Exception): @@ -71,22 +70,24 @@ def _run(self) -> None: if not sessions: return LOGGER.info('Found %d sessions that require action', len(sessions)) - inventory_cache = Inventory() - for data in sessions: - session = Session(data, inventory_cache, self.bos_client) - session.setup(self.max_batch_size) + with Inventory() as inventory_cache: + for data in sessions: + session = Session(data, inventory_cache, self.client.bos, + self.HSMState) + session.setup(self.max_batch_size) def _get_pending_sessions(self): - return self.bos_client.sessions.get_sessions(status='pending') + return self.client.bos.sessions.get_sessions(status='pending') class Session: - def __init__(self, data, inventory_cache, bos_client): + def __init__(self, data, inventory_cache, bos_client, hsm_state): self.session_data = data self.inventory = inventory_cache self.bos_client = bos_client self._template = None + self.HSMState = hsm_state @property def name(self): @@ -104,8 +105,8 @@ def operation_type(self): def template(self): if not self._template: template_name = self.session_data.get('template_name') - self._template = self.bos_client.session_templates.get_session_template(template_name, - self.tenant) + self._template = self.bos_client.session_templates.get_session_template( + template_name, self.tenant) return self._template def setup(self, max_batch_size: int): @@ -130,14 +131,16 @@ def _setup_components(self, max_batch_size: int): else: state = self._generate_desired_state(boot_set) for component_id in components: - data.append(self._operate(component_id, copy.deepcopy(state))) + data.append( + self._operate(component_id, copy.deepcopy(state))) all_component_ids += components if not all_component_ids: raise SessionSetupException("No nodes were found to act upon.") except Exception as err: raise SessionSetupException(err) from err # No exception raised by previous block - self._log(LOGGER.info, 'Found %d components that require updates', len(data)) + self._log(LOGGER.info, 'Found %d components that require updates', + len(data)) for chunk in chunk_components(data, max_batch_size): self._log(LOGGER.debug, f'Updated components: {chunk}') self.bos_client.components.update_components(chunk) @@ -158,20 +161,25 @@ def _get_boot_set_component_list(self, boot_set) -> Set[str]: # Populate from node_groups for group_name in boot_set.get('node_groups', []): if group_name not in self.inventory.groups: - self._log(LOGGER.warning, f"No hardware matching label {group_name}") + self._log(LOGGER.warning, + f"No hardware matching label {group_name}") continue nodes |= self.inventory.groups[group_name] # Populate from node_roles_groups for role_name in boot_set.get('node_roles_groups', []): if role_name not in self.inventory.roles: - self._log(LOGGER.warning, f"No hardware matching role {role_name}") + self._log(LOGGER.warning, + f"No hardware matching role {role_name}") continue nodes |= self.inventory.roles[role_name] if not nodes: - self._log(LOGGER.warning, - "After populating node list, before any filtering, no nodes to act upon.") + self._log( + LOGGER.warning, + "After populating node list, before any filtering, no nodes to act upon." + ) return nodes - self._log(LOGGER.debug, "Before any limiting or filtering, %d nodes to act upon.", + self._log(LOGGER.debug, + "Before any limiting or filtering, %d nodes to act upon.", len(nodes)) # Filter out any nodes that do not match the boot set architecture desired; boot sets that # do not have a specified arch are considered 'X86' nodes. @@ -212,14 +220,18 @@ def _apply_arch(self, nodes, arch): valid_archs = set([arch]) if arch == 'X86': valid_archs.add('UNKNOWN') - hsm_filter = HSMState() + hsm_filter = self.HSMState() nodes = set(hsm_filter.filter_by_arch(nodes, valid_archs)) if not nodes: - self._log(LOGGER.warning, - "After filtering for architecture, no nodes remain to act upon.") + self._log( + LOGGER.warning, + "After filtering for architecture, no nodes remain to act upon." + ) else: - self._log(LOGGER.debug, - "After filtering for architecture, %d nodes remain to act upon.", len(nodes)) + self._log( + LOGGER.debug, + "After filtering for architecture, %d nodes remain to act upon.", + len(nodes)) return nodes def _apply_include_disabled(self, nodes): @@ -231,14 +243,17 @@ def _apply_include_disabled(self, nodes): if include_disabled: # Nodes disabled in HSM may be included, so no filtering is required return nodes - hsmfilter = HSMState(enabled=True) + hsmfilter = self.HSMState(enabled=True) nodes = set(hsmfilter._filter(list(nodes))) if not nodes: - self._log(LOGGER.warning, - "After removing disabled nodes, no nodes remain to act upon.") + self._log( + LOGGER.warning, + "After removing disabled nodes, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After removing disabled nodes, %d nodes remain to act upon.", - len(nodes)) + self._log( + LOGGER.debug, + "After removing disabled nodes, %d nodes remain to act upon.", + len(nodes)) return nodes def _apply_limit(self, nodes): @@ -266,9 +281,11 @@ def _apply_limit(self, nodes): limit_node_set = op(limit_nodes) nodes = nodes.intersection(limit_node_set) if not nodes: - self._log(LOGGER.warning, "After applying limit, no nodes remain to act upon.") + self._log(LOGGER.warning, + "After applying limit, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After applying limit, %d nodes remain to act upon.", + self._log(LOGGER.debug, + "After applying limit, %d nodes remain to act upon.", len(nodes)) return nodes @@ -282,16 +299,24 @@ def _apply_tenant_limit(self, nodes): raise SessionSetupException(str(e)) from e nodes = nodes.intersection(tenant_limit) if not nodes: - self._log(LOGGER.warning, "After applying tenant limit, no nodes remain to act upon.") + self._log( + LOGGER.warning, + "After applying tenant limit, no nodes remain to act upon.") else: - self._log(LOGGER.debug, "After applying tenant limit, %d nodes remain to act upon.", - len(nodes)) + self._log( + LOGGER.debug, + "After applying tenant limit, %d nodes remain to act upon.", + len(nodes)) return nodes def _mark_running(self, component_ids): self.bos_client.sessions.update_session( - self.name, self.tenant, - {'status': {'status': 'running'}, "components": ",".join(component_ids)}) + self.name, self.tenant, { + 'status': { + 'status': 'running' + }, + "components": ",".join(component_ids) + }) self._log(LOGGER.info, 'Session is running') def _mark_failed(self, err): @@ -299,8 +324,10 @@ def _mark_failed(self, err): Input: err (string): The error that prevented the session from running """ - self.bos_client.sessions.update_session( - self.name, self.tenant, {'status': {'error': err}}) + self.bos_client.sessions.update_session(self.name, self.tenant, + {'status': { + 'error': err + }}) sco = SessionCompletionOperator() sco._mark_session_complete(self.name, self.tenant) self._log(LOGGER.info, 'Session %s has failed.', self.name) @@ -317,7 +344,7 @@ def _operate(self, component_id, state): data["staged_state"]["session"] = self.name else: data["desired_state"] = state - if self.operation_type == "reboot" : + if self.operation_type == "reboot": data["actual_state"] = EMPTY_ACTUAL_STATE data["session"] = self.name data["enabled"] = True @@ -346,11 +373,14 @@ def _get_state_from_boot_set(self, boot_set): image_metadata = BootImageMetaDataFactory(boot_set)() artifact_info = image_metadata.artifact_summary boot_artifacts['kernel'] = artifact_info['kernel'] - boot_artifacts['initrd'] = image_metadata.initrd.get("link", {}).get("path", "") - boot_artifacts['kernel_parameters'] = self.assemble_kernel_boot_parameters(boot_set, - artifact_info) + boot_artifacts['initrd'] = image_metadata.initrd.get("link", {}).get( + "path", "") + boot_artifacts[ + 'kernel_parameters'] = self.assemble_kernel_boot_parameters( + boot_set, artifact_info) state['boot_artifacts'] = boot_artifacts - state['configuration'] = self._get_configuration_from_boot_set(boot_set) + state['configuration'] = self._get_configuration_from_boot_set( + boot_set) return state def _get_configuration_from_boot_set(self, boot_set: dict): @@ -402,26 +432,28 @@ def assemble_kernel_boot_parameters(self, boot_set, artifact_info): boot_param_pieces = [] # Parameters from the image itself if the parameters exist. - if (artifact_info.get('boot_parameters') is not None and - artifact_info.get('boot_parameters_etag') is not None): - self._log(LOGGER.info, - "++ _get_s3_download_url %s with etag %s.", - artifact_info['boot_parameters'], - artifact_info['boot_parameters_etag']) + if (artifact_info.get('boot_parameters') is not None + and artifact_info.get('boot_parameters_etag') is not None): + self._log(LOGGER.info, "++ _get_s3_download_url %s with etag %s.", + artifact_info['boot_parameters'], + artifact_info['boot_parameters_etag']) try: s3_obj = S3Object(artifact_info['boot_parameters'], artifact_info['boot_parameters_etag']) image_kernel_parameters_object = s3_obj.object - parameters_raw = image_kernel_parameters_object['Body'].read().decode('utf-8') + parameters_raw = image_kernel_parameters_object['Body'].read( + ).decode('utf-8') image_kernel_parameters = parameters_raw.split() if image_kernel_parameters: boot_param_pieces.extend(image_kernel_parameters) - except (ClientError, UnicodeDecodeError, S3ObjectNotFound) as error: - self._log(LOGGER.error, - "Error reading file %s; no kernel boot parameters obtained from image", - artifact_info['boot_parameters']) + except (ClientError, UnicodeDecodeError, + S3ObjectNotFound) as error: + self._log( + LOGGER.error, + "Error reading file %s; no kernel boot parameters obtained from image", + artifact_info['boot_parameters']) self._log(LOGGER.error, exc_type_msg(error)) raise @@ -440,7 +472,8 @@ def assemble_kernel_boot_parameters(self, boot_set, artifact_info): boot_param_pieces.append(nmd_parameters) # Add the bos actual state ttl value so nodes know how frequently to report - boot_param_pieces.append(f'bos_update_frequency={options.component_actual_state_ttl}') + boot_param_pieces.append( + f'bos_update_frequency={options.component_actual_state_ttl}') return ' '.join(boot_param_pieces) diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index 83980f07..315347d2 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -27,12 +27,10 @@ from bos.common.values import Phase, Status, Action, EMPTY_ACTUAL_STATE from bos.operators.base import BaseOperator, main from bos.operators.filters import DesiredBootStateIsOff, BootArtifactStatesMatch, \ - DesiredConfigurationIsNone, DesiredConfigurationSetInCFS, LastActionIs, TimeSinceLastAction -from bos.operators.utils.clients.bos.options import options -from bos.operators.utils.clients.pcs import node_to_powerstate -from bos.operators.utils.clients.cfs import get_components as get_cfs_components + DesiredConfigurationIsNone, LastActionIs, TimeSinceLastAction +from bos.common.clients.bos.options import options -LOGGER = logging.getLogger('bos.operators.status') +LOGGER = logging.getLogger(__name__) class StatusOperator(BaseOperator): @@ -46,13 +44,16 @@ def __init__(self): # Reuse filter code self.desired_boot_state_is_off = DesiredBootStateIsOff()._match self.boot_artifact_states_match = BootArtifactStatesMatch()._match - self.desired_configuration_is_none = DesiredConfigurationIsNone()._match - self.desired_configuration_set_in_cfs = DesiredConfigurationSetInCFS()._match + self.desired_configuration_is_none = DesiredConfigurationIsNone( + )._match self.last_action_is_power_on = LastActionIs(Action.power_on)._match self.boot_wait_time_elapsed = TimeSinceLastAction( - seconds=options.max_boot_wait_time)._match + seconds=options.max_boot_wait_time)._match self.power_on_wait_time_elapsed = TimeSinceLastAction( - seconds=options.max_power_on_wait_time)._match + seconds=options.max_power_on_wait_time)._match + + def desired_configuration_set_in_cfs(self, *args, **kwargs): + return self.DesiredConfigurationSetInCFS()._match(*args, **kwargs) @property def name(self): @@ -70,11 +71,12 @@ def _act(self, components): def _run(self) -> None: """ A single pass of detecting and acting on components """ - components = self.bos_client.components.get_components(enabled=True) + components = self.client.bos.components.get_components(enabled=True) if not components: LOGGER.debug('No enabled components found') return - LOGGER.debug('Found %d components that require action', len(components)) + LOGGER.debug('Found %d components that require action', + len(components)) for chunk in self._chunk_components(components): self._run_on_chunk(chunk) @@ -84,35 +86,37 @@ def _run_on_chunk(self, components) -> None: """ LOGGER.debug("Processing %d components", len(components)) component_ids = [component['id'] for component in components] - power_states = node_to_powerstate(component_ids) + power_states = self.client.pcs.power_status.node_to_powerstate( + component_ids) cfs_states = self._get_cfs_components() updated_components = [] # Recreate these filters to pull in the latest options values self.boot_wait_time_elapsed = TimeSinceLastAction( - seconds=options.max_boot_wait_time)._match + seconds=options.max_boot_wait_time)._match self.power_on_wait_time_elapsed = TimeSinceLastAction( - seconds=options.max_power_on_wait_time)._match + seconds=options.max_power_on_wait_time)._match for component in components: updated_component = self._check_status( - component, power_states.get(component['id']), cfs_states.get(component['id'])) + component, power_states.get(component['id']), + cfs_states.get(component['id'])) if updated_component: updated_components.append(updated_component) if not updated_components: LOGGER.debug('No components require status updates') return - LOGGER.info('Found %d components that require status updates', len(updated_components)) + LOGGER.info('Found %d components that require status updates', + len(updated_components)) LOGGER.debug('Updated components: %s', updated_components) - self.bos_client.components.update_components(updated_components) + self.client.bos.components.update_components(updated_components) - @staticmethod - def _get_cfs_components(): + def _get_cfs_components(self): """ Gets all the components from CFS. We used to get only the components of interest, but that caused an HTTP request that was longer than uwsgi could handle when the number of nodes was very large. Requesting all components means none need to be specified in the request. """ - cfs_data = get_cfs_components() + cfs_data = self.client.cfs.components.get_components() cfs_states = {} for component in cfs_data: cfs_states[component['id']] = component @@ -125,9 +129,8 @@ def _check_status(self, component, power_state, cfs_component): """ error = None if power_state and cfs_component: - phase, override, disable, error, action_failed = self._calculate_status(component, - power_state, - cfs_component) + phase, override, disable, error, action_failed = self._calculate_status( + component, power_state, cfs_component) else: # If the component cannot be found in pcs or cfs phase = Phase.none @@ -170,8 +173,8 @@ def _check_status(self, component, power_state, cfs_component): if error and error != component.get('error', ''): updated_component['error'] = error update = True - if action_failed and action_failed != component.get('last_action', {}).get('failed', - False): + if action_failed and action_failed != component.get( + 'last_action', {}).get('failed', False): updated_component['last_action'] = {} updated_component['last_action']['failed'] = True update = True @@ -204,15 +207,17 @@ def _calculate_status(self, component, power_state, cfs_component): phase = Phase.none disable = True # Successful state - desired and actual state are off else: - if self.last_action_is_power_on(component) and self.power_on_wait_time_elapsed( - component): + if self.last_action_is_power_on( + component) and self.power_on_wait_time_elapsed( + component): action_failed = True phase = Phase.powering_on else: if self.desired_boot_state_is_off(component): phase = Phase.powering_off elif self.boot_artifact_states_match(component): - if not self.desired_configuration_set_in_cfs(component, cfs_component): + if not self.desired_configuration_set_in_cfs( + component, cfs_component): phase = Phase.configuring elif self.desired_configuration_is_none(component): # Successful state - booted with the correct artifacts, @@ -220,7 +225,8 @@ def _calculate_status(self, component, power_state, cfs_component): phase = Phase.none disable = True else: - cfs_status = cfs_component.get('configuration_status', '').lower() + cfs_status = cfs_component.get('configuration_status', + '').lower() if cfs_status == 'configured': # Successful state - booted with the correct artifacts and configured phase = Phase.none @@ -238,11 +244,13 @@ def _calculate_status(self, component, power_state, cfs_component): phase = Phase.configuring disable = True override = Status.failed - error = ('cfs is not reporting a valid configuration status for ' - f'this component: {cfs_status}') + error = ( + 'cfs is not reporting a valid configuration status for ' + f'this component: {cfs_status}') else: - if self.last_action_is_power_on(component) and not self.boot_wait_time_elapsed( - component): + if self.last_action_is_power_on( + component + ) and not self.boot_wait_time_elapsed(component): phase = Phase.powering_on else: # Includes both power-off for restarts and ready-recovery scenario diff --git a/src/bos/operators/utils/boot_image_metadata/factory.py b/src/bos/operators/utils/boot_image_metadata/factory.py index 0227540a..c922a585 100644 --- a/src/bos/operators/utils/boot_image_metadata/factory.py +++ b/src/bos/operators/utils/boot_image_metadata/factory.py @@ -25,7 +25,7 @@ from bos.operators.utils.boot_image_metadata.s3_boot_image_metadata import S3BootImageMetaData -LOGGER = logging.getLogger('bos.operators.utils.boot_image_metadata.factory') +LOGGER = logging.getLogger(__name__) class BootImageMetaDataUnknown(Exception): @@ -33,18 +33,22 @@ class BootImageMetaDataUnknown(Exception): Raised when a user requests a Provider provisioning mechanism that is not known """ + class BootImageMetaDataFactory: """ Conditionally create new instances of the BootImageMetadata based on the type of the BootImageMetaData specified """ + def __init__(self, boot_set: dict): self.boot_set = boot_set def __call__(self): path_type = self.boot_set.get('type', None) if not path_type: - raise BootImageMetaDataUnknown(f"No path type set in boot set: {self.boot_set}") + raise BootImageMetaDataUnknown( + f"No path type set in boot set: {self.boot_set}") if path_type == 's3': return S3BootImageMetaData(self.boot_set) - raise BootImageMetaDataUnknown(f"No BootImageMetaData class for type {path_type}") + raise BootImageMetaDataUnknown( + f"No BootImageMetaData class for type {path_type}") diff --git a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py index 163d0b56..87fbeb34 100644 --- a/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py +++ b/src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py @@ -27,10 +27,10 @@ from bos.common.utils import exc_type_msg from bos.operators.utils.boot_image_metadata import BootImageMetaData, BootImageMetaDataBadRead -from bos.operators.utils.clients.s3 import S3BootArtifacts, S3MissingConfiguration, S3Url, \ +from bos.common.clients.s3 import S3BootArtifacts, S3MissingConfiguration, S3Url, \ ArtifactNotFound -LOGGER = logging.getLogger('bos.operators.utils.boot_image_metadata.s3_boot_image_metadata') +LOGGER = logging.getLogger(__name__) class S3BootImageMetaData(BootImageMetaData): @@ -61,11 +61,13 @@ def __init__(self, boot_set: dict): except ArtifactNotFound as err: LOGGER.warning(exc_type_msg(err)) try: - self.artifact_summary['boot_parameters'] = self.boot_parameters_path + self.artifact_summary[ + 'boot_parameters'] = self.boot_parameters_path except ArtifactNotFound as err: LOGGER.warning(exc_type_msg(err)) try: - self.artifact_summary['boot_parameters_etag'] = self.boot_parameters_etag + self.artifact_summary[ + 'boot_parameters_etag'] = self.boot_parameters_etag except ArtifactNotFound as err: LOGGER.warning(exc_type_msg(err)) @@ -81,8 +83,8 @@ def metadata(self): try: return self.boot_artifacts.manifest_json except (ClientError, S3MissingConfiguration) as error: - LOGGER.error("Unable to read %s -- Error: %s", self._boot_set.get('path', ''), - exc_type_msg(error)) + LOGGER.error("Unable to read %s -- Error: %s", + self._boot_set.get('path', ''), exc_type_msg(error)) raise BootImageMetaDataBadRead(error) from error @property diff --git a/src/bos/operators/utils/clients/bos/base.py b/src/bos/operators/utils/clients/bos/base.py deleted file mode 100644 index 11d05d29..00000000 --- a/src/bos/operators/utils/clients/bos/base.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import json -import logging -from requests.exceptions import HTTPError, ConnectionError -from urllib3.exceptions import MaxRetryError - -from bos.common.tenant_utils import get_new_tenant_header -from bos.common.utils import PROTOCOL, exc_type_msg, requests_retry_session - -LOGGER = logging.getLogger('bos.operators.utils.clients.bos.base') - -API_VERSION = 'v2' -SERVICE_NAME = 'cray-bos' -BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{API_VERSION}" - - -def log_call_errors(func): - - def wrap(*args, **kwargs): - try: - result = func(*args, **kwargs) - return result - except (ConnectionError, MaxRetryError) as e: - LOGGER.error("Unable to connect to BOS: %s", exc_type_msg(e)) - raise e - except HTTPError as e: - LOGGER.error("Unexpected response from BOS: %s", exc_type_msg(e)) - raise e - except json.JSONDecodeError as e: - LOGGER.error("Non-JSON response from BOS: %s", exc_type_msg(e)) - raise e - - return wrap - - -class BaseBosEndpoint: - """ - This base class provides generic access to the BOS API. - The individual endpoint needs to be overridden for a specific endpoint. - """ - ENDPOINT = '' - - def __init__(self): - self.base_url = f"{BASE_ENDPOINT}/{self.ENDPOINT}" - self.session = requests_retry_session() - - @log_call_errors - def get_item(self, item_id): - """Get information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("GET %s", url) - response = self.session.get(url) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def get_items(self, **kwargs): - """Get information for all BOS items""" - LOGGER.debug("GET %s with params=%s", self.base_url, kwargs) - response = self.session.get(self.base_url, params=kwargs) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def update_item(self, item_id, data): - """Update information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("PATCH %s with body=%s", url, data) - response = self.session.patch(url, json=data) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def update_items(self, data): - """Update information for multiple BOS items""" - LOGGER.debug("PATCH %s with body=%s", self.base_url, data) - response = self.session.patch(self.base_url, json=data) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def put_items(self, data): - """Put information for multiple BOS Items""" - LOGGER.debug("PUT %s with body=%s", self.base_url, data) - response = self.session.put(self.base_url, json=data) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def delete_items(self, **kwargs): - """Delete information for multiple BOS items""" - LOGGER.debug("DELETE %s with params=%s", self.base_url, kwargs) - response = self.session.delete(self.base_url, params=kwargs) - response.raise_for_status() - return json.loads(response.text) if response.text else None - - -class BaseBosTenantAwareEndpoint(BaseBosEndpoint): - """ - This base class provides generic access to the BOS API for tenant aware endpoints. - The individual endpoint needs to be overridden for a specific endpoint. - """ - - @log_call_errors - def get_item(self, item_id, tenant): - """Get information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("GET %s for tenant=%s", url, tenant) - response = self.session.get(url, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def get_items(self, **kwargs): - """Get information for all BOS items""" - headers = None - if "tenant" in kwargs: - tenant = kwargs.pop("tenant") - headers = get_new_tenant_header(tenant) - LOGGER.debug("GET %s for tenant=%s with params=%s", self.base_url, tenant, kwargs) - else: - LOGGER.debug("GET %s with params=%s", self.base_url, kwargs) - response = self.session.get(self.base_url, params=kwargs, headers=headers) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def update_item(self, item_id, tenant, data): - """Update information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("PATCH %s for tenant=%s with body=%s", url, tenant, data) - response = self.session.patch(url, json=data, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def update_items(self, tenant, data): - """Update information for multiple BOS items""" - LOGGER.debug("PATCH %s for tenant=%s with body=%s", self.base_url, tenant, data) - response = self.session.patch(self.base_url, json=data, - headers=get_new_tenant_header(tenant)) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def put_items(self, tenant, data): - """Put information for multiple BOS items""" - LOGGER.debug("PUT %s for tenant=%s with body=%s", self.base_url, tenant, data) - response = self.session.put(self.base_url, json=data, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def delete_items(self, **kwargs): - """Delete information for multiple BOS items""" - headers = None - if "tenant" in kwargs: - tenant = kwargs.pop("tenant") - headers = get_new_tenant_header(tenant) - LOGGER.debug("DELETE %s for tenant=%s with params=%s", self.base_url, tenant, kwargs) - else: - LOGGER.debug("DELETE %s with params=%s", self.base_url, kwargs) - response = self.session.delete(self.base_url, params=kwargs, headers=headers) - response.raise_for_status() - return json.loads(response.text) if response.text else None diff --git a/src/bos/operators/utils/clients/bos/sessions_status.py b/src/bos/operators/utils/clients/bos/sessions_status.py deleted file mode 100644 index 4fae6b4d..00000000 --- a/src/bos/operators/utils/clients/bos/sessions_status.py +++ /dev/null @@ -1,63 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import json -import logging - -from bos.common.tenant_utils import get_new_tenant_header -from bos.common.utils import requests_retry_session -from .base import BASE_ENDPOINT, log_call_errors - -LOGGER = logging.getLogger('bos.operators.utils.clients.bos.sessions_status') - - -class SessionStatusEndpoint: - ENDPOINT = 'sessions' - - def __init__(self): - self.base_url = f"{BASE_ENDPOINT}/{self.ENDPOINT}" - - @log_call_errors - def get_session_status(self, session_id, tenant): - """Get information for a single BOS item""" - url = self.base_url + '/' + session_id + '/status' - session = requests_retry_session() - LOGGER.debug("GET %s for tenant=%s", url, tenant) - response = session.get(url, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def post_session_status(self, session_id, tenant): - """ - Post information for a single BOS Session status. - This basically saves the BOS Session status to the database. - """ - session = requests_retry_session() - url = self.base_url + '/' + session_id + '/status' - LOGGER.debug("POST %s for tenant=%s", url, tenant) - response = session.post(url, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - items = json.loads(response.text) - return items diff --git a/src/bos/operators/utils/clients/bss.py b/src/bos/operators/utils/clients/bss.py deleted file mode 100644 index 0b33c2f2..00000000 --- a/src/bos/operators/utils/clients/bss.py +++ /dev/null @@ -1,87 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2019-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import logging -import json - -from requests.exceptions import HTTPError - -from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL - -LOGGER = logging.getLogger(__name__) -SERVICE_NAME = 'cray-bss' -ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/boot/v1" - - -def set_bss(node_set, kernel_params, kernel, initrd, session=None): - ''' - Tell the Boot Script Service (BSS) which boot artifacts are associated - with each node. - - Currently, this is biased towards 'hosts' (i.e. xnames) rather than - NIDS. - - Args: - node_set (set): A list of nodes to assign the boot artifacts to - kernel_params (string): Kernel parameters to assign to the node - kernel (string): The kernel to assign to the node - initrd (string): The initrd to assign to the node - session (requests Session instance): An existing session to use - - Returns: - The response from BSS. - - Raises: - KeyError -- If the boot_artifacts does not find either the initrd - or kernel keys, this error is raised. - ValueError -- if the kernel_parameters contains an 'initrd' - requests.exceptions.HTTPError -- An HTTP error encountered while - communicating with the - Hardware State Manager - ''' - if not node_set: - # Cannot simply return if no nodes are specified, as this function - # is intended to return the response object from BSS. - # Accordingly, an Exception is raised. - raise Exception("set_bss called with empty node_set") - - session = session or requests_retry_session() - LOGGER.info("Params: %s", kernel_params) - url = f"{ENDPOINT}/bootparameters" - - # Assignment payload - payload = {"hosts": list(node_set), - "params": kernel_params, - "kernel": kernel, - "initrd": initrd} - - LOGGER.debug("PUT %s for hosts %s", url, node_set) - try: - resp = session.put(url, data=json.dumps(payload), verify=False) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", resp.status_code, - resp.reason, compact_response_text(resp.text)) - resp.raise_for_status() - return resp - except HTTPError as err: - LOGGER.error(exc_type_msg(err)) - raise diff --git a/src/bos/operators/utils/clients/cfs.py b/src/bos/operators/utils/clients/cfs.py deleted file mode 100644 index 5ac48265..00000000 --- a/src/bos/operators/utils/clients/cfs.py +++ /dev/null @@ -1,140 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -from collections import defaultdict -import logging -from bos.operators.utils.clients.bos.options import options -from requests.exceptions import HTTPError - -from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL - -SERVICE_NAME = 'cray-cfs-api' -BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/v3" -COMPONENTS_ENDPOINT = f"{BASE_ENDPOINT}/components" - -LOGGER = logging.getLogger('bos.operators.utils.clients.cfs') - -GET_BATCH_SIZE = 200 -PATCH_BATCH_SIZE = 1000 - - -def get_components(session=None, **params): - """ - Makes GET request for CFS components. - Performs additional requests to get additional pages of components, if - needed. - Returns the list of CFS components - """ - if not session: - session = requests_retry_session(read_timeout=options.cfs_read_timeout) # pylint: disable=redundant-keyword-arg - component_list = [] - while params is not None: - LOGGER.debug("GET %s with params=%s", COMPONENTS_ENDPOINT, params) - response = session.get(COMPONENTS_ENDPOINT, params=params) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - except HTTPError as err: - LOGGER.error("Failed getting nodes from CFS: %s", exc_type_msg(err)) - raise - response_json = response.json() - new_components = response_json["components"] - LOGGER.debug("Query returned %d components", len(new_components)) - component_list.extend(new_components) - params = response_json["next"] - LOGGER.debug("Returning %d components from CFS", len(component_list)) - return component_list - - -def patch_components(data, session=None): - if not data: - LOGGER.warning("patch_components called without data; returning without action.") - return - if not session: - session = requests_retry_session(read_timeout=options.cfs_read_timeout) # pylint: disable=redundant-keyword-arg - LOGGER.debug("PATCH %s with body=%s", COMPONENTS_ENDPOINT, data) - response = session.patch(COMPONENTS_ENDPOINT, json=data) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - except HTTPError as err: - LOGGER.error("Failed asking CFS to configure nodes: %s", exc_type_msg(err)) - raise - - -def get_components_from_id_list(id_list): - if not id_list: - LOGGER.warning("get_components_from_id_list called without IDs; returning without action.") - return [] - LOGGER.debug("get_components_from_id_list called with %d IDs", len(id_list)) - session = requests_retry_session(read_timeout=options.cfs_read_timeout) # pylint: disable=redundant-keyword-arg - component_list = [] - while id_list: - next_batch = id_list[:GET_BATCH_SIZE] - next_comps = get_components(session=session, ids=','.join(next_batch)) - component_list.extend(next_comps) - id_list = id_list[GET_BATCH_SIZE:] - LOGGER.debug("get_components_from_id_list returning a total of %d components from CFS", - len(component_list)) - return component_list - - -def patch_desired_config(node_ids, desired_config, enabled=False, tags=None, clear_state=False): - if not node_ids: - LOGGER.warning("patch_desired_config called without IDs; returning without action.") - return - LOGGER.debug("patch_desired_config called on %d IDs with desired_config=%s enabled=%s tags=%s" - " clear_state=%s", len(node_ids), desired_config, enabled, tags, clear_state) - session = requests_retry_session(read_timeout=options.cfs_read_timeout) # pylint: disable=redundant-keyword-arg - node_patch = { - 'enabled': enabled, - 'desired_config': desired_config, - 'tags': tags if tags else {} - } - data={ "patch": node_patch, "filters": {} } - if clear_state: - node_patch['state'] = [] - while node_ids: - data["filters"]["ids"] = ','.join(node_ids[:PATCH_BATCH_SIZE]) - patch_components(data=data, session=session) - node_ids = node_ids[PATCH_BATCH_SIZE:] - - -def set_cfs(components, enabled, clear_state=False): - if not components: - LOGGER.warning("set_cfs called without components; returning without action.") - return - LOGGER.debug("set_cfs called on %d components with enabled=%s clear_state=%s", len(components), - enabled, clear_state) - configurations = defaultdict(list) - for component in components: - config_name = component.get('desired_state', {}).get('configuration', '') - bos_session = component.get('session') - key = (config_name, bos_session) - configurations[key].append(component['id']) - for key, ids in configurations.items(): - config_name, bos_session = key - patch_desired_config(ids, config_name, enabled=enabled, - tags={'bos_session': bos_session}, clear_state=clear_state) diff --git a/src/bos/operators/utils/clients/hsm.py b/src/bos/operators/utils/clients/hsm.py deleted file mode 100644 index f6cb3d95..00000000 --- a/src/bos/operators/utils/clients/hsm.py +++ /dev/null @@ -1,239 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import json -import logging -import os -from collections import defaultdict -from requests.exceptions import HTTPError, ConnectionError -from urllib3.exceptions import MaxRetryError - -from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL - -SERVICE_NAME = 'cray-smd' -BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/hsm/v2/" -ENDPOINT = os.path.join(BASE_ENDPOINT, 'State/Components/Query') -VERIFY = True - -LOGGER = logging.getLogger('bos.operators.utils.clients.hsm') - - -class HWStateManagerException(Exception): - """ - An error unique to interacting with the HWStateManager service; - should the service be unable to fulfill a given request (timeout, - no components, service 503s, etc.); this exception is raised. It is - intended to be further subclassed for more specific kinds of errors - in the future should they arise. - """ - - -def read_all_node_xnames(): - """ - Queries HSM for the full set of xname components that - have been discovered; return these as a set. - """ - session = requests_retry_session() - endpoint = f'{BASE_ENDPOINT}/State/Components/' - LOGGER.debug("GET %s", endpoint) - try: - response = session.get(endpoint) - except ConnectionError as ce: - LOGGER.error("Unable to contact HSM service: %s", exc_type_msg(ce)) - raise HWStateManagerException(ce) from ce - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - except (HTTPError, MaxRetryError) as hpe: - LOGGER.error("Unexpected response from HSM: %s (%s)", response, exc_type_msg(hpe)) - raise HWStateManagerException(hpe) from hpe - try: - json_body = json.loads(response.text) - except json.JSONDecodeError as jde: - LOGGER.error("Non-JSON response from HSM: %s", response.text) - raise HWStateManagerException(jde) from jde - try: - return {component['ID'] for component in json_body['Components'] - if component.get('Type', None) == 'Node'} - except KeyError as ke: - LOGGER.error("Unexpected API response from HSM: %s", exc_type_msg(ke)) - raise HWStateManagerException(ke) from ke - - -def get_components(node_list, enabled=None) -> dict[str,list[dict]]: - """ - Get information for all list components HSM - - :return the HSM components - :rtype Dictionary containing a 'Components' key whose value is a list - containing each component, where each component is itself represented by a - dictionary. - - Here is an example of the returned values. - { - "Components": [ - { - "ID": "x3000c0s19b1n0", - "Type": "Node", - "State": "Ready", - "Flag": "OK", - "Enabled": true, - "Role": "Compute", - "NID": 1, - "NetType": "Sling", - "Arch": "X86", - "Class": "River" - }, - { - "ID": "x3000c0s19b2n0", - "Type": "Node", - "State": "Ready", - "Flag": "OK", - "Enabled": true, - "Role": "Compute", - "NID": 1, - "NetType": "Sling", - "Arch": "X86", - "Class": "River" - } - ] - } - """ - if not node_list: - LOGGER.warning("hsm.get_components called with empty node list") - return {'Components': []} - session = requests_retry_session() - try: - payload = {'ComponentIDs': node_list} - if enabled is not None: - payload['enabled'] = [str(enabled)] - LOGGER.debug("POST %s with body=%s", ENDPOINT, payload) - response = session.post(ENDPOINT, json=payload) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - response.raise_for_status() - components = json.loads(response.text) - except (ConnectionError, MaxRetryError) as e: - LOGGER.error("Unable to connect to HSM: %s", exc_type_msg(e)) - raise e - except HTTPError as e: - LOGGER.error("Unexpected response from HSM: %s", exc_type_msg(e)) - raise e - except json.JSONDecodeError as e: - LOGGER.error("Non-JSON response from HSM: %s", exc_type_msg(e)) - raise e - return components - - -class Inventory: - """ - Inventory handles the generation of a hardware inventory in a similar manner to how the - dynamic inventory is generated for CFS. To reduce the number of calls to HSM, everything is - cached for repeated checks, stored both as overall inventory and separate group types to allow - use in finding BOS's base list of nodes, and lazily loaded to prevent extra calls when no limit - is used. - """ - - def __init__(self, partition=None): - self._partition = partition # Can be specified to limit to roles/components query - self._inventory = None - self._groups = None - self._partitions = None - self._roles = None - self._session = None - - @property - def groups(self): - if self._groups is None: - data = self.get('groups') - groups = {} - for group in data: - groups[group['label']] = set(group.get('members', {}).get('ids', [])) - self._groups = groups - return self._groups - - @property - def partitions(self): - if self._partitions is None: - data = self.get('partitions') - partitions = {} - for partition in data: - partitions[partition['name']] = set(partition.get('members', {}).get('ids', [])) - self._partitions = partitions - return self._partitions - - @property - def roles(self): - if self._roles is None: - params = {} - if self._partition: - params['partition'] = self._partition - data = self.get('State/Components', params=params) - roles = defaultdict(set) - for component in data['Components']: - role='' - if 'Role' in component: - role = str(component['Role']) - roles[role].add(component['ID']) - if 'SubRole' in component: - subrole = role + '_' + str(component['SubRole']) - roles[subrole].add(component['ID']) - self._roles = roles - return self._roles - - @property - def inventory(self): - if self._inventory is None: - inventory = {} - inventory.update(self.groups) - inventory.update(self.partitions) - inventory.update(self.roles) - self._inventory = inventory - LOGGER.info(self._inventory) - return self._inventory - - def __contains__(self, key): - return key in self.inventory - - def __getitem__(self, key): - return self.inventory[key] - - def get(self, path, params=None): - url = os.path.join(BASE_ENDPOINT, path) - if self._session is None: - self._session = requests_retry_session() - try: - LOGGER.debug("HSM Inventory: GET %s with params=%s", url, params) - response = self._session.get(url, params=params, verify=VERIFY) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - response.raise_for_status() - except HTTPError as err: - LOGGER.error("Failed to get '%s': %s", url, exc_type_msg(err)) - raise - try: - return response.json() - except ValueError: - LOGGER.error("Couldn't parse a JSON response: %s", response.text) - raise diff --git a/src/bos/operators/utils/clients/ims.py b/src/bos/operators/utils/clients/ims.py deleted file mode 100644 index 8fc93335..00000000 --- a/src/bos/operators/utils/clients/ims.py +++ /dev/null @@ -1,179 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# - -import logging -import re -from requests.exceptions import HTTPError -from requests.sessions import Session as RequestsSession - -from bos.common.utils import compact_response_text, exc_type_msg, requests_retry_session, PROTOCOL -from bos.operators.utils.clients.s3 import S3Url - -SERVICE_NAME = 'cray-ims' -IMS_VERSION = 'v3' -BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{IMS_VERSION}" -IMAGES_ENDPOINT = f"{BASE_ENDPOINT}/images" - -LOGGER = logging.getLogger('bos.operators.utils.clients.ims') -IMS_TAG_OPERATIONS = ['set', 'remove'] - -# Making minimal assumptions about the IMS ID itself, this pattern just makes sure that the -# S3 key is some string, then a /, then at least one more character. -IMS_S3_KEY_RE = r'^([^/]+)/.+' -IMS_S3_KEY_RE_PROG = re.compile(IMS_S3_KEY_RE) - -# If an IMS image does not have the arch field, default to x86_64 for purposes of -# backward-compatibility -DEFAULT_IMS_IMAGE_ARCH = 'x86_64' - - -class TagFailure(Exception): - pass - - -class ImageNotFound(Exception): - """ - Raised if querying IMS for an image and it is not found - """ - def __init__(self, image_id: str): - super().__init__(f"IMS image id '{image_id}' does not exist in IMS") - - -def get_image(image_id: str, session: RequestsSession|None=None) -> dict: - """ - Queries IMS to retrieve the specified image and return it. - If the image does not exist, raise ImageNotFound. - Other errors (like a failure to query IMS) will result in appropriate exceptions being raised. - """ - if not session: - session = requests_retry_session() - url=f"{IMAGES_ENDPOINT}/{image_id}" - LOGGER.debug("GET %s", url) - try: - response = session.get(url) - except Exception as err: - LOGGER.error("Exception during GET request to %s: %s", url, exc_type_msg(err)) - raise - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - except HTTPError as err: - msg = f"Failed asking IMS to get image {image_id}: {exc_type_msg(err)}" - if response.status_code == 404: - # If it's not found, we just log it as a warning, because we may be - # okay with that -- that will be for the caller to decide - LOGGER.warning(msg) - raise ImageNotFound(image_id) from err - LOGGER.error(msg) - raise - try: - return response.json() - except Exception as err: - LOGGER.error("Failed decoding JSON response from getting IMS image %s: %s", image_id, - exc_type_msg(err)) - raise - - -def patch_image(image_id: str, data: dict, session: RequestsSession|None=None) -> None: - if not data: - LOGGER.warning("patch_image called without data; returning without action.") - return - if not session: - session = requests_retry_session() - url=f"{IMAGES_ENDPOINT}/{image_id}" - LOGGER.debug("PATCH %s with body=%s", url, data) - response = session.patch(url, json=data) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - except HTTPError as err: - LOGGER.error("Failed asking IMS to patch image %s: %s", image_id, exc_type_msg(err)) - if response.status_code == 404: - raise ImageNotFound(image_id) from err - raise - - -def tag_image(image_id: str, operation: str, key: str, value: str=None, - session: RequestsSession|None=None) -> None: - if operation not in IMS_TAG_OPERATIONS: - msg = f"{operation} not valid. Expecting one of {IMS_TAG_OPERATIONS}" - LOGGER.error(msg) - raise TagFailure(msg) - - if not key: - msg = f"key must exist: {key}" - LOGGER.error(msg) - raise TagFailure(msg) - - if value: - LOGGER.debug("Patching image %s %sing key: %s value: %s", image_id, operation, key, value) - else: - LOGGER.debug("Patching image %s %sing key: %s", image_id, operation, key) - - if not session: - session = requests_retry_session() - - data = { - "metadata": { - "operation": operation, - "key": key, - "value": value - } - } - patch_image(image_id=image_id, data=data, session=session) - - -def get_ims_id_from_s3_url(s3_url: S3Url) -> str|None: - """ - If the s3_url matches the expected format of an IMS image path, then return the IMS image ID. - Otherwise return None. - """ - try: - return IMS_S3_KEY_RE_PROG.match(s3_url.key).group(1) - except (AttributeError, IndexError): - return None - - -def get_arch_from_image_data(image_data: dict) -> str: - """ - Returns the value of the 'arch' field in the image data - If it is not present, logs a warning and returns the default value - """ - try: - arch = image_data['arch'] - except KeyError: - LOGGER.warning("Defaulting to '%s' because 'arch' not set in IMS image data: %s", - DEFAULT_IMS_IMAGE_ARCH, image_data) - return DEFAULT_IMS_IMAGE_ARCH - except Exception as err: - LOGGER.error("Unexpected error parsing IMS image data (%s): %s", exc_type_msg(err), - image_data) - raise - if arch: - return arch - LOGGER.warning("Defaulting to '%s' because 'arch' set to null value in IMS image data: %s", - DEFAULT_IMS_IMAGE_ARCH, image_data) - return DEFAULT_IMS_IMAGE_ARCH diff --git a/src/bos/operators/utils/clients/pcs.py b/src/bos/operators/utils/clients/pcs.py deleted file mode 100644 index fddafdd7..00000000 --- a/src/bos/operators/utils/clients/pcs.py +++ /dev/null @@ -1,291 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# - -# This client wrapper is derived from the PCS source spec, as provided by: -# https://github.com/Cray-HPE/hms-power-control/blob/develop/api/swagger.yaml - -import logging -import json -from collections import defaultdict - -import requests - -from bos.common.utils import compact_response_text, requests_retry_session, PROTOCOL - -SERVICE_NAME = 'cray-power-control' -POWER_CONTROL_VERSION = 'v1' -ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}" -POWER_STATUS_ENDPOINT = f'{ENDPOINT}/power-status' -TRANSITION_ENDPOINT = f"{ENDPOINT}/transitions" - -LOGGER = logging.getLogger('bos.operators.utils.clients.pcs') - - -class PowerControlException(Exception): - """ - Interaction with PCS resulted in a known failure. - """ - - -class PowerControlSyntaxException(Exception): - """ - A class of error raised when interacting with PCS in an unsupported way. - """ - - -class PowerControlTimeoutException(PowerControlException): - """ - Raised when a call to PowerControl exceeded total time to complete. - """ - - -class PowerControlComponentsEmptyException(Exception): - """ - Raised when one of the PCS utility functions that requires a non-empty - list of components is passed an empty component list. This will only - happen in the case of a programming bug. - - This exception is not raised for functions that require a node list - but that are able to return a sensible object to the caller that - indicates nothing has been done. For example, the status function. - This exception is instead used for functions that will fail if they run - with an empty node list, but which cannot return an appropriate - "no-op" value to the caller. - """ - -def _power_status(xname=None, power_state_filter=None, management_state_filter=None, - session=None): - """ - This is the one to one implementation to the underlying power control get query. - For reasons of compatibility with existing calls into older power control APIs, - existing functions call into this function to preserve the existing functionality - already implemented. - - Users may specify one of three filters, and a power_status_all (PCS defined schema) - is returned. Users may elect to use a previously generated session in order to query - the results. If not, the default requests retry session will be generated. - - Per the spec, a power_status_all is returned. power_status_all is an array of power - statuses. - """ - session = session or requests_retry_session() - params = {} - if xname: - params['xname'] = xname - if power_state_filter: - assert power_state_filter.lower() in set(['on','off','undefined']) - params['powerStateFilter'] = power_state_filter.lower() - if management_state_filter: - assert management_state_filter in set(['available', 'unavailable']) - params['managementStateFilter'] = management_state_filter.lower() - # PCS added the POST option for this endpoint in app version 2.3.0 - # (chart versions 2.0.8 and 2.1.5) - LOGGER.debug("POST %s with body=%s", POWER_STATUS_ENDPOINT, params) - response = session.post(POWER_STATUS_ENDPOINT, json=params) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - if not response.ok: - raise PowerControlException(f"Non-2XX response ({response.status_code}) to " - f"power_status query; {response.reason} " - f"{compact_response_text(response.text)}") - except requests.exceptions.HTTPError as err: - raise PowerControlException(err) from err - try: - return response.json() - except json.JSONDecodeError as jde: - raise PowerControlException(jde) from jde - -def status(nodes, session=None, **kwargs): - """ - For a given iterable of nodes, represented by xnames, query PCS for - the power status. Return a dictionary of nodes that have - been bucketed by status. - - Args: - nodes (list): Nodes to get status for - session (session object): An already instantiated session - kwargs: Any additional args used for filtering when calling _power_status. - This can be useful if you want to limit your search to only available or unavailable nodes, - and allows a more future-proof way of handling arguments to PCS as a catch-all parameter. - - Returns: - status_dict (dict): Keys are different states; values are a literal set of nodes. - Nodes with errors associated with them are saved with the error value as a - status key. - - Raises: - PowerControlException: Any non-nominal response from PCS. - JSONDecodeError: Error decoding the PCS response - """ - status_bucket = defaultdict(set) - if not nodes: - LOGGER.warning("status called without nodes; returning without action.") - return status_bucket - session = session or requests_retry_session() - power_status_all = _power_status(xname=list(nodes), session=session, **kwargs) - for power_status_entry in power_status_all['status']: - # If the returned xname has an error, it itself is the status regardless of - # what the powerState field suggests. This is a major departure from how CAPMC - # handled errors. - xname = power_status_entry.get('xname', '') - if power_status_entry['error']: - status_bucket[power_status_entry['error']].add(xname) - continue - power_status = power_status_entry.get('powerState', '').lower() - if not all([power_status, xname]): - continue - status_bucket[power_status].add(xname) - return status_bucket - -def node_to_powerstate(nodes, session=None, **kwargs): - """ - For an iterable of nodes ; return a dictionary that maps to the current power state for - the node in question. - """ - power_states = {} - if not nodes: - LOGGER.warning("node_to_powerstate called without nodes; returning without action.") - return power_states - session = session or requests_retry_session() - status_bucket = status(nodes, session, **kwargs) - for pstatus, nodeset in status_bucket.items(): - for node in nodeset: - power_states[node] = pstatus - return power_states - -def _transition_create(xnames, operation, task_deadline_minutes=None, deputy_key=None, - session=None): - """ - Interact with PCS to create a request to transition one or more xnames. The transition - operation indicates what the desired operation should be, which is a string value containing - one or more of the supported transition names for the given hardware, e.g. 'on', 'off', or - 'force-off'. - - Once created, one of two responses are returned. A 2XX response results in a - transition_start_output object, or, an invalid request results in a 4XX and subsequent - raised PCS exception. - - Args: - xnames: an iterable of xnames - operation: A string/enum for what the nodes should transition to - task_deadline_minutes: How long should PCS operate on the nodes to bring them to complete - the operation; typecast to an integer value. - deputy_key: An optional string value that can be used to further handle instructing PCS - to perform state transitions on behalf of a known existing reservation. - session: An already existing session to use with PCS, if any - - Returns: - A transition_start_output object, which is a record for the transition that was created. - The most important field of which is the 'transitionID' value, which allows subsequent - follow-on to the created request. - - Raises: - PowerControlException: Any non-nominal response from PCS, typically as a result of an - unexpected payload response, or a failure to create a transition record. - PowerControlComponentsEmptyException: No xnames specified - """ - if not xnames: - raise PowerControlComponentsEmptyException( - f"_transition_create called with no xnames! (operation={operation})") - session = session or requests_retry_session() - try: - assert operation in {'On', 'Off', 'Soft-Off', 'Soft-Restart', 'Hard-Restart', 'Init', - 'Force-Off'} - except AssertionError as err: - raise PowerControlSyntaxException( - f"Operation '{operation}' is not supported or implemented.") from err - params = {'location': [], 'operation': operation} - if task_deadline_minutes: - params['taskDeadlineMinutes'] = int(task_deadline_minutes) - for xname in xnames: - reserved_location = {'xname': xname} - if deputy_key: - reserved_location['deputyKey'] = deputy_key - params['location'].append(reserved_location) - LOGGER.debug("POST %s with body=%s", TRANSITION_ENDPOINT, params) - response = session.post(TRANSITION_ENDPOINT, json=params) - LOGGER.debug("Response status code=%d, reason=%s, body=%s", response.status_code, - response.reason, compact_response_text(response.text)) - try: - response.raise_for_status() - if not response.ok: - raise PowerControlException(f"Non-2XX response ({response.status_code}) to " - f"{operation} power transition creation; " - f"{response.reason} " - f"{compact_response_text(response.text)}") - - except requests.exceptions.HTTPError as err: - raise PowerControlException(err) from err - try: - return response.json() - except json.decoder.JSONDecodeError as jde: - raise PowerControlException(jde) from jde - - -def power_on(nodes, session=None, task_deadline_minutes=1, **kwargs): - """ - Sends a request to PCS for transitioning nodes in question to a powered on state. - Returns: A JSON parsed object response from PCS, which includes the created request ID. - """ - if not nodes: - raise PowerControlComponentsEmptyException("power_on called with no nodes!") - session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='On', - task_deadline_minutes=task_deadline_minutes, - session=session, **kwargs) -def power_off(nodes, session=None, task_deadline_minutes=1, **kwargs): - """ - Sends a request to PCS for transitioning nodes in question to a powered off state (graceful). - Returns: A JSON parsed object response from PCS, which includes the created request ID. - """ - if not nodes: - raise PowerControlComponentsEmptyException("power_off called with no nodes!") - session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='Off', - task_deadline_minutes=task_deadline_minutes, - session=session, **kwargs) -def soft_off(nodes, session=None, task_deadline_minutes=1, **kwargs): - """ - Sends a request to PCS for transitioning nodes in question to a powered off state (graceful). - Returns: A JSON parsed object response from PCS, which includes the created request ID. - """ - if not nodes: - raise PowerControlComponentsEmptyException("soft_off called with no nodes!") - session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='Soft-Off', - task_deadline_minutes=task_deadline_minutes, - session=session, **kwargs) -def force_off(nodes, session=None, task_deadline_minutes=1, **kwargs): - """ - Sends a request to PCS for transitioning nodes in question to a powered off state (forceful). - Returns: A JSON parsed object response from PCS, which includes the created request ID. - """ - if not nodes: - raise PowerControlComponentsEmptyException("force_off called with no nodes!") - session = session or requests_retry_session() - return _transition_create(xnames=nodes, operation='Force-Off', - task_deadline_minutes=task_deadline_minutes, - session=session, **kwargs) diff --git a/src/bos/operators/utils/liveness/__main__.py b/src/bos/operators/utils/liveness/__main__.py index d843afd2..0bc91bff 100644 --- a/src/bos/operators/utils/liveness/__main__.py +++ b/src/bos/operators/utils/liveness/__main__.py @@ -28,14 +28,14 @@ from bos.operators.utils.liveness import TIMESTAMP_PATH from bos.operators.utils.liveness.timestamp import Timestamp - -LOGGER = logging.getLogger('bos.operators.utils.liveness.main') +LOGGER = logging.getLogger(__name__) DEFAULT_LOG_LEVEL = logging.INFO def setup_logging(): log_format = "%(asctime)-15s - %(levelname)-7s - %(name)s - %(message)s" - requested_log_level = os.environ.get('BOS_OPERATOR_LOG_LEVEL', DEFAULT_LOG_LEVEL) + requested_log_level = os.environ.get('BOS_OPERATOR_LOG_LEVEL', + DEFAULT_LOG_LEVEL) log_level = logging.getLevelName(requested_log_level) logging.basicConfig(level=log_level, format=log_format) @@ -44,7 +44,8 @@ def setup_logging(): setup_logging() timestamp = Timestamp.byref(TIMESTAMP_PATH) if timestamp.alive: - LOGGER.info("%s is considered valid; the application is alive!", timestamp) + LOGGER.info("%s is considered valid; the application is alive!", + timestamp) sys.exit(0) else: LOGGER.warning("Timestamp is no longer considered valid.") diff --git a/src/bos/server/__main__.py b/src/bos/server/__main__.py index 8c934b81..fc6312e9 100644 --- a/src/bos/server/__main__.py +++ b/src/bos/server/__main__.py @@ -32,7 +32,7 @@ from bos.server.controllers.v2 import options from bos.server.encoder import JSONEncoder -LOGGER = logging.getLogger('bos.__main__') +LOGGER = logging.getLogger(__name__) def create_app(): @@ -46,8 +46,7 @@ def create_app(): app = connexion.App(__name__, specification_dir='./openapi/') app.app.json_encoder = JSONEncoder app.add_api('openapi.yaml', - arguments={'title': - 'Cray Boot Orchestration Service'}, + arguments={'title': 'Cray Boot Orchestration Service'}, base_path='/') return app diff --git a/src/bos/server/controllers/base.py b/src/bos/server/controllers/base.py index 11cf66a2..c689fa7a 100644 --- a/src/bos/server/controllers/base.py +++ b/src/bos/server/controllers/base.py @@ -27,13 +27,11 @@ from bos.server.controllers.v2 import base as v2_base -LOGGER = logging.getLogger('bos.server.controllers.base') +LOGGER = logging.getLogger(__name__) def root_get(): """ Get a list of supported versions """ LOGGER.info('in get_versions') - versions = [ - v2_base.calc_version(details=False) - ] + versions = [v2_base.calc_version(details=False)] return versions, 200 diff --git a/src/bos/server/controllers/utils.py b/src/bos/server/controllers/utils.py index 1203ac0b..ec466a33 100644 --- a/src/bos/server/controllers/utils.py +++ b/src/bos/server/controllers/utils.py @@ -28,7 +28,7 @@ import connexion import flask -LOGGER = logging.getLogger('bos.server.controllers.utils') +LOGGER = logging.getLogger(__name__) def url_for(endpoint, **values): @@ -64,10 +64,10 @@ def url_for(endpoint, **values): # Request was proxied, so update the path with the proxy path. parts = urlparse(url) - parts = ( - parts.scheme, parts.netloc, - '/'.join([proxy_path.rstrip('/'), parts.path.lstrip('/')]), - parts.params, parts.query, parts.fragment) + parts = (parts.scheme, parts.netloc, + '/'.join([proxy_path.rstrip('/'), + parts.path.lstrip('/') + ]), parts.params, parts.query, parts.fragment) return urlunparse(parts) # TODO(CASMCMS-1869): there might be a better way to do this by overriding diff --git a/src/bos/server/controllers/v2/base.py b/src/bos/server/controllers/v2/base.py index 5cfda958..98d3b42d 100644 --- a/src/bos/server/controllers/v2/base.py +++ b/src/bos/server/controllers/v2/base.py @@ -28,7 +28,7 @@ from bos.server.controllers.utils import url_for from bos.server.models import Version, Link -LOGGER = logging.getLogger('bos.server.controllers.v2.base') +LOGGER = logging.getLogger(__name__) def calc_version(details): @@ -55,7 +55,8 @@ def calc_version(details): major, minor, patch = openapispec_map['info']['version'].split('.') return Version(major=major, minor=minor, patch=patch, links=links) except IOError as e: - LOGGER.exception('error opening "%s" file: %s', openapispec_f, exc_type_msg(e)) + LOGGER.exception('error opening "%s" file: %s', openapispec_f, + exc_type_msg(e)) raise diff --git a/src/bos/server/controllers/v2/boot_set/artifacts.py b/src/bos/server/controllers/v2/boot_set/artifacts.py index 087b850e..73562024 100644 --- a/src/bos/server/controllers/v2/boot_set/artifacts.py +++ b/src/bos/server/controllers/v2/boot_set/artifacts.py @@ -24,7 +24,7 @@ from bos.common.utils import exc_type_msg from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory -from bos.operators.utils.clients.s3 import S3Object, ArtifactNotFound +from bos.common.clients.s3 import S3Object, ArtifactNotFound from .defs import LOGGER from .exceptions import BootSetError, BootSetWarning @@ -35,13 +35,14 @@ def validate_boot_artifacts(bs: dict): try: image_metadata = BootImageMetaDataFactory(bs)() except Exception as err: - raise BootSetError(f"Can't find boot artifacts. Error: {exc_type_msg(err)}") from err + raise BootSetError( + f"Can't find boot artifacts. Error: {exc_type_msg(err)}") from err # Check boot artifacts' S3 headers for boot_artifact in ["kernel"]: try: artifact = getattr(image_metadata.boot_artifacts, boot_artifact) - path = artifact ['link']['path'] + path = artifact['link']['path'] etag = artifact['link']['etag'] obj = S3Object(path, etag) _ = obj.object_header @@ -55,7 +56,7 @@ def validate_boot_artifacts(bs: dict): artifact = getattr(image_metadata.boot_artifacts, boot_artifact) if not artifact: raise ArtifactNotFound() - path = artifact ['link']['path'] + path = artifact['link']['path'] etag = artifact['link']['etag'] obj = S3Object(path, etag) _ = obj.object_header diff --git a/src/bos/server/controllers/v2/boot_set/defs.py b/src/bos/server/controllers/v2/boot_set/defs.py index 5a21a33a..dcf48b28 100644 --- a/src/bos/server/controllers/v2/boot_set/defs.py +++ b/src/bos/server/controllers/v2/boot_set/defs.py @@ -25,7 +25,8 @@ from enum import IntEnum import logging -LOGGER = logging.getLogger('bos.server.controllers.v2.boot_set') +LOGGER = logging.getLogger(__name__) + # Use IntEnum to allow for inequalities class BootSetStatus(IntEnum): @@ -36,7 +37,8 @@ class BootSetStatus(IntEnum): WARNING = 1 ERROR = 2 + # Valid boot sets are required to have at least one of these fields -HARDWARE_SPECIFIER_FIELDS = ( "node_list", "node_roles_groups", "node_groups" ) +HARDWARE_SPECIFIER_FIELDS = ("node_list", "node_roles_groups", "node_groups") DEFAULT_ARCH = "X86" diff --git a/src/bos/server/controllers/v2/boot_set/ims.py b/src/bos/server/controllers/v2/boot_set/ims.py index 0df4a165..55e0b40a 100644 --- a/src/bos/server/controllers/v2/boot_set/ims.py +++ b/src/bos/server/controllers/v2/boot_set/ims.py @@ -22,10 +22,10 @@ # OTHER DEALINGS IN THE SOFTWARE. # -from bos.common.utils import exc_type_msg, requests_retry_session -from bos.operators.utils.clients.ims import get_arch_from_image_data, get_image, \ +from bos.common.utils import exc_type_msg +from bos.common.clients.ims import get_arch_from_image_data, IMSClient, \ get_ims_id_from_s3_url, ImageNotFound -from bos.operators.utils.clients.s3 import S3Url +from bos.common.clients.s3 import S3Url from bos.server.controllers.v2.options import OptionsData from .defs import DEFAULT_ARCH @@ -34,11 +34,7 @@ # Mapping from BOS boot set arch values to expected IMS image arch values # Omits BOS Other value, since there is no corresponding IMS image arch value -EXPECTED_IMS_ARCH = { - "ARM": "aarch64", - "Unknown": "x86_64", - "X86": "x86_64" -} +EXPECTED_IMS_ARCH = {"ARM": "aarch64", "Unknown": "x86_64", "X86": "x86_64"} def validate_ims_boot_image(bs: dict, options_data: OptionsData) -> None: @@ -91,7 +87,8 @@ def validate_ims_boot_image(bs: dict, options_data: OptionsData) -> None: raise BootSetWarning(str(err)) from err if EXPECTED_IMS_ARCH[bs_arch] != ims_image_arch: - raise BootSetArchMismatch(bs_arch=bs_arch, expected_ims_arch=EXPECTED_IMS_ARCH[bs_arch], + raise BootSetArchMismatch(bs_arch=bs_arch, + expected_ims_arch=EXPECTED_IMS_ARCH[bs_arch], actual_ims_arch=ims_image_arch) @@ -106,18 +103,18 @@ def get_ims_image_id(path: str) -> str: ims_id = get_ims_id_from_s3_url(s3_url) if ims_id: return ims_id - raise NonImsImage(f"Boot artifact S3 URL '{s3_url.url}' doesn't follow convention " - "for IMS images") + raise NonImsImage( + f"Boot artifact S3 URL '{s3_url.url}' doesn't follow convention " + "for IMS images") -def get_ims_image_data(ims_id: str, num_retries: int|None=None) -> dict: +def get_ims_image_data(ims_id: str, num_retries: int | None = None) -> dict: """ Query IMS to get the image data and return it, or raise an exception. """ - kwargs = { "image_id": ims_id } + kwargs = {} if num_retries is not None: - # A pylint bug generates a false positive error for this call - # https://github.com/pylint-dev/pylint/issues/2271 - kwargs['session'] = requests_retry_session(retries=4) # pylint: disable=redundant-keyword-arg - return get_image(**kwargs) + kwargs = {'retries': num_retries} + with IMSClient(**kwargs) as ims_client: + return ims_client.images.get_image(ims_id) diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 0c03b406..949f1ce3 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -21,6 +21,7 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +from functools import partial import logging import connexion @@ -34,81 +35,137 @@ from bos.server.dbs.boot_artifacts import get_boot_artifacts, BssTokenUnknown from bos.server.utils import get_request_json -LOGGER = logging.getLogger('bos.server.controllers.v2.components') +LOGGER = logging.getLogger(__name__) DB = dbutils.get_wrapper(db='components') SESSIONS_DB = dbutils.get_wrapper(db='sessions') @tenant_error_handler @dbutils.redis_error_handler -def get_v2_components(ids="", enabled=None, session=None, staged_session=None, phase=None, - status=None): +def get_v2_components(ids="", + enabled=None, + session=None, + staged_session=None, + phase=None, + status=None, + start_after_id=None, + page_size=None): """Used by the GET /components API operation Allows filtering using a comma separated list of ids. """ - LOGGER.debug("GET /v2/components invoked get_v2_components with ids=%s enabled=%s session=%s " - "staged_session=%s phase=%s status=%s", ids, enabled, session, staged_session, - phase, status) + LOGGER.debug( + "GET /v2/components invoked get_v2_components with ids=%s enabled=%s session=%s " + "staged_session=%s phase=%s status=%s", ids, enabled, session, + staged_session, phase, status) id_list = [] if ids: try: id_list = ids.split(',') except Exception as err: LOGGER.error("Error parsing component IDs: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the ids provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the ids provided.", + detail=str(err)) tenant = get_tenant_from_header() - LOGGER.debug("GET /v2/components for tenant=%s with %d IDs specified", tenant, len(id_list)) - response = get_v2_components_data(id_list=id_list, enabled=enabled, session=session, + LOGGER.debug("GET /v2/components for tenant=%s with %d IDs specified", + tenant, len(id_list)) + response = get_v2_components_data(id_list=id_list, + enabled=enabled, + session=session, staged_session=staged_session, - phase=phase, status=status, tenant=tenant) - LOGGER.debug("GET /v2/components returning data for tenant=%s on %d components", tenant, - len(response)) - for component in response: - del_timestamp(component) + phase=phase, + status=status, + tenant=tenant, + start_after_id=start_after_id, + page_size=page_size or 0, + delete_timestamp=True) + LOGGER.debug( + "GET /v2/components returning data for tenant=%s on %d components", + tenant, len(response)) return response, 200 -def get_v2_components_data(id_list=None, enabled=None, session=None, staged_session=None, - phase=None, status=None, tenant=None): +def get_v2_components_data(id_list=None, + enabled=None, + session=None, + staged_session=None, + phase=None, + status=None, + tenant=None, + start_after_id=None, + page_size=0, + delete_timestamp: bool = False): """Used by the GET /components API operation Allows filtering using a comma separated list of ids. """ - response = [] - if id_list: - for component_id in id_list: - data = DB.get(component_id) - if data: - response.append(data) + if any([id_list, enabled, session, staged_session, phase, status, tenant]): + tenant_components = None if not tenant else get_tenant_component_set( + tenant) + _component_filter_func = partial(_filter_component, + id_list=id_list or None, + enabled=enabled, + session=session or None, + staged_session=staged_session or None, + phase=phase or None, + status=status or None, + tenant_components=tenant_components + or None) + elif delete_timestamp: + _component_filter_func = partial(_set_status, + delete_timestamp=delete_timestamp) else: - # TODO: On large scale systems, this response may be too large - # and require paging to be implemented - response = DB.get_all() - # The status must be set before using _matches_filter as the status is one of the - # matching criteria. - response = [_set_status(r) for r in response if r] - if enabled is not None or session is not None or staged_session is not None or \ - phase is not None or status is not None: - response = [r for r in response if _matches_filter(r, enabled, session, staged_session, - phase, status)] - if tenant: - tenant_components = get_tenant_component_set(tenant) - limited_response = [component for component in response - if component["id"] in tenant_components] - response = limited_response - return response + _component_filter_func = _set_status + + return DB.get_all_filtered(filter_func=_component_filter_func, + start_after_key=start_after_id, + page_size=page_size) + + +def _filter_component(data: dict, + id_list=None, + enabled=None, + session=None, + staged_session=None, + phase=None, + status=None, + tenant_components=None, + delete_timestamp: bool = False) -> dict | None: + # Do all of the checks we can before calculating status, to avoid doing it needlessly + if id_list is not None and data["id"] not in id_list: + return None + if tenant_components is not None and data["id"] not in tenant_components: + return None + if enabled is not None and data.get('enabled', None) != enabled: + return None + if session is not None and data.get('session', None) != session: + return None + if staged_session is not None and \ + data.get('staged_state', {}).get('session', None) != staged_session: + return None + updated_data = _set_status(data) + + status_data = updated_data.get('status') + if phase is not None and status_data.get('phase') != phase: + return None + if status is not None and status_data.get('status') not in status.split( + ','): + return None + if delete_timestamp: + del_timestamp(updated_data) + return updated_data -def _set_status(data): +def _set_status(data, delete_timestamp: bool = False): """ This sets the status field of the overall status. """ if "status" not in data: data["status"] = {"phase": "", "status_override": ""} data['status']['status'] = _calculate_status(data) + if delete_timestamp: + del_timestamp(data) return data @@ -127,11 +184,13 @@ def _calculate_status(data): phase = status_data.get('phase', '') component = data.get('id', '') - last_action = data.get('last_action', {}).get('action', '') + last_action_dict = data.get('last_action', {}) + last_action = last_action_dict.get('action', '') status = status = Status.stable if phase == Phase.powering_on: - if last_action == Action.power_on and not data.get('last_action', {}).get('failed', False): + if last_action == Action.power_on and not last_action_dict.get( + 'failed', False): status = Status.power_on_called else: status = Status.power_on_pending @@ -145,27 +204,11 @@ def _calculate_status(data): elif phase == Phase.configuring: status = Status.configuring - LOGGER.debug("Component: %s Last action: %s Phase: %s Status: %s", component, last_action, - phase, status) + LOGGER.debug("Component: %s Last action: %s Phase: %s Status: %s", + component, last_action, phase, status) return status -def _matches_filter(data, enabled, session, staged_session, phase, status): - if enabled is not None and data.get('enabled', None) != enabled: - return False - if session is not None and data.get('session', None) != session: - return False - if staged_session is not None and \ - data.get('staged_state', {}).get('session', None) != staged_session: - return False - status_data = data.get('status') - if phase is not None and status_data.get('phase') != phase: - return False - if status is not None and status_data.get('status') not in status.split(','): - return False - return True - - @dbutils.redis_error_handler def put_v2_components(): """Used by the PUT /components API operation""" @@ -174,9 +217,9 @@ def put_v2_components(): data = get_request_json() except Exception as err: LOGGER.error("Error parsing PUT request data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) components = [] for component_data in data: @@ -184,8 +227,10 @@ def put_v2_components(): component_id = component_data['id'] except KeyError: return connexion.problem( - status=400, title="Required field missing.", - detail="At least one component is missing the required 'id' field") + status=400, + title="Required field missing.", + detail= + "At least one component is missing the required 'id' field") components.append((component_id, component_data)) response = [] for component_id, component_data in components: @@ -203,9 +248,9 @@ def patch_v2_components(): data = get_request_json() except Exception as err: LOGGER.error("Error parsing PATCH request data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) if isinstance(data, list): return patch_v2_components_list(data) @@ -214,33 +259,38 @@ def patch_v2_components(): LOGGER.error("Unexpected data type %s", str(type(data))) return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=f"Unexpected data type {type(data).__name__}") + status=400, + title="Error parsing the data provided.", + detail=f"Unexpected data type {type(data).__name__}") def patch_v2_components_list(data): try: - LOGGER.debug("patch_v2_components_list: %d components specified", len(data)) + LOGGER.debug("patch_v2_components_list: %d components specified", + len(data)) components = [] for component_data in data: component_id = component_data['id'] - if component_id not in DB or not _is_valid_tenant_component(component_id): + if component_id not in DB or not _is_valid_tenant_component( + component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( - status=404, title="Component not found.", + status=404, + title="Component not found.", detail=f"Component {component_id} could not be found") components.append((component_id, component_data)) except Exception as err: LOGGER.error("Error loading component data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) response = [] for component_id, component_data in components: if "id" in component_data: del component_data["id"] component_data = _set_auto_fields(component_data) - response.append(DB.patch(component_id, component_data, _update_handler)) + response.append(DB.patch(component_id, component_data, + _update_handler)) return response, 200 @@ -250,34 +300,41 @@ def patch_v2_components_dict(data): session = filters.get("session", None) if ids and session: LOGGER.warning("Multiple filters provided") - return connexion.problem( - status=400, title="Only one filter may be provided.", - detail="Only one filter may be provided.") + return connexion.problem(status=400, + title="Only one filter may be provided.", + detail="Only one filter may be provided.") if ids: try: id_list = ids.split(',') except Exception as err: - LOGGER.error("Error parsing the IDs provided: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the ids provided.", - detail=str(err)) + LOGGER.error("Error parsing the IDs provided: %s", + exc_type_msg(err)) + return connexion.problem(status=400, + title="Error parsing the ids provided.", + detail=str(err)) # Make sure all of the components exist and belong to this tenant (if any) - LOGGER.debug("patch_v2_components_dict: %d IDs specified", len(id_list)) + LOGGER.debug("patch_v2_components_dict: %d IDs specified", + len(id_list)) for component_id in id_list: - if component_id not in DB or not _is_valid_tenant_component(component_id): + if component_id not in DB or not _is_valid_tenant_component( + component_id): return connexion.problem( - status=404, title="Component not found.", + status=404, + title="Component not found.", detail=f"Component {component_id} could not be found") elif session: - id_list = [component["id"] for component in get_v2_components_data( - session=session, - tenant=get_tenant_from_header())] - LOGGER.debug("patch_v2_components_dict: %d IDs found for specified session", len(id_list)) + id_list = [ + component["id"] for component in get_v2_components_data( + session=session, tenant=get_tenant_from_header()) + ] + LOGGER.debug( + "patch_v2_components_dict: %d IDs found for specified session", + len(id_list)) else: LOGGER.warning("No filter provided") - return connexion.problem( - status=400, title="Exactly one filter must be provided.", - detail="Exactly one filter may be provided.") + return connexion.problem(status=400, + title="Exactly one filter must be provided.", + detail="Exactly one filter may be provided.") response = [] patch = data.get("patch") if "id" in patch: @@ -292,11 +349,13 @@ def patch_v2_components_dict(data): @dbutils.redis_error_handler def get_v2_component(component_id): """Used by the GET /components/{component_id} API operation""" - LOGGER.debug("GET /v2/components/%s invoked get_v2_component", component_id) + LOGGER.debug("GET /v2/components/%s invoked get_v2_component", + component_id) if component_id not in DB or not _is_valid_tenant_component(component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( - status=404, title="Component not found.", + status=404, + title="Component not found.", detail=f"Component {component_id} could not be found") component = DB.get(component_id) component = _set_status(component) @@ -307,14 +366,16 @@ def get_v2_component(component_id): @dbutils.redis_error_handler def put_v2_component(component_id): """Used by the PUT /components/{component_id} API operation""" - LOGGER.debug("PUT /v2/components/%s invoked put_v2_component", component_id) + LOGGER.debug("PUT /v2/components/%s invoked put_v2_component", + component_id) try: data = get_request_json() except Exception as err: - LOGGER.error("Error parsing PUT '%s' request data: %s", component_id, exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + LOGGER.error("Error parsing PUT '%s' request data: %s", component_id, + exc_type_msg(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) data['id'] = component_id data = _set_auto_fields(data) @@ -325,26 +386,31 @@ def put_v2_component(component_id): @dbutils.redis_error_handler def patch_v2_component(component_id): """Used by the PATCH /components/{component_id} API operation""" - LOGGER.debug("PATCH /v2/components/%s invoked patch_v2_component", component_id) + LOGGER.debug("PATCH /v2/components/%s invoked patch_v2_component", + component_id) try: data = get_request_json() except Exception as err: - LOGGER.error("Error parsing PATCH '%s' request data: %s", component_id, exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + LOGGER.error("Error parsing PATCH '%s' request data: %s", component_id, + exc_type_msg(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) if component_id not in DB or not _is_valid_tenant_component(component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( - status=404, title="Component not found.", + status=404, + title="Component not found.", detail=f"Component {component_id} could not be found") - if "actual_state" in data and not validate_actual_state_change_is_allowed(component_id): + if "actual_state" in data and not validate_actual_state_change_is_allowed( + component_id): LOGGER.warning("Not able to update actual state") return connexion.problem( - status=409, title="Actual state can not be updated.", + status=409, + title="Actual state can not be updated.", detail="BOS is currently changing the state of the node," - " and the actual state can not be accurately recorded") + " and the actual state can not be accurately recorded") if "id" in data: del data["id"] data = _set_auto_fields(data) @@ -373,11 +439,13 @@ def validate_actual_state_change_is_allowed(component_id): @dbutils.redis_error_handler def delete_v2_component(component_id): """Used by the DELETE /components/{component_id} API operation""" - LOGGER.debug("DELETE /v2/components/%s invoked delete_v2_component", component_id) + LOGGER.debug("DELETE /v2/components/%s invoked delete_v2_component", + component_id) if component_id not in DB or not _is_valid_tenant_component(component_id): LOGGER.warning("Component %s could not be found", component_id) return connexion.problem( - status=404, title="Component not found.", + status=404, + title="Component not found.", detail=f"Component {component_id} could not be found") return DB.delete(component_id), 204 @@ -391,9 +459,9 @@ def post_v2_apply_staged(): data = get_request_json() except Exception as err: LOGGER.error("Error parsing POST request data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) response = {"succeeded": [], "failed": [], "ignored": []} # Obtain latest desired behavior for how to clear staging information @@ -411,13 +479,14 @@ def post_v2_apply_staged(): response["ignored"].append(xname) except Exception: LOGGER.exception( - "An error was encountered while attempting to apply stage for node %s", xname) + "An error was encountered while attempting to apply stage for node %s", + xname) response["failed"].append(xname) except Exception as err: LOGGER.error("Error parsing request data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) return response, 200 @@ -470,24 +539,29 @@ def _set_state_from_staged(data): staged_state = data.get("staged_state", {}) staged_session_id_sans_tenant = staged_state.get("session", "") tenant = get_tenant_from_header() - staged_session_id = get_tenant_aware_key(staged_session_id_sans_tenant, tenant) + staged_session_id = get_tenant_aware_key(staged_session_id_sans_tenant, + tenant) if staged_session_id not in SESSIONS_DB: raise Exception("Staged session no longer exists") session = SESSIONS_DB.get(staged_session_id) operation = session["operation"] if operation == "shutdown": if any(staged_state.get("boot_artifacts", {}).values()): - raise Exception("Staged operation is shutdown but boot artifact have been specified") + raise Exception( + "Staged operation is shutdown but boot artifact have been specified" + ) _copy_staged_to_desired(data) elif operation == "boot": if not all(staged_state.get("boot_artifacts", {}).values()): raise Exception( - "Staged operation is boot but some boot artifacts have not been specified") + "Staged operation is boot but some boot artifacts have not been specified" + ) _copy_staged_to_desired(data) elif operation == "reboot": if not all(staged_state.get("boot_artifacts", {}).values()): raise Exception( - "Staged operation is reboot but some boot artifacts have not been specified") + "Staged operation is reboot but some boot artifacts have not been specified" + ) _copy_staged_to_desired(data) data["actual_state"] = { "boot_artifacts": EMPTY_BOOT_ARTIFACTS, @@ -538,7 +612,8 @@ def _populate_boot_artifacts(data): # Populate the boot artifacts using the bss_token if token: try: - data['actual_state']['boot_artifacts'] = get_boot_artifacts(token) + data['actual_state']['boot_artifacts'] = get_boot_artifacts( + token) except BssTokenUnknown: LOGGER.warning("Reported BSS Token '%s' is unknown.", token) return data @@ -559,9 +634,11 @@ def del_timestamp(data: dict): def _set_last_updated(data): timestamp = get_current_timestamp() - for section in ['actual_state', 'desired_state', 'staged_state', 'last_action']: - if section in data and isinstance(data[section], - dict) and data[section].keys() != {"bss_token"}: + for section in [ + 'actual_state', 'desired_state', 'staged_state', 'last_action' + ]: + if section in data and isinstance( + data[section], dict) and data[section].keys() != {"bss_token"}: data[section]['last_updated'] = timestamp return data diff --git a/src/bos/server/controllers/v2/healthz.py b/src/bos/server/controllers/v2/healthz.py index 166c0aea..83b941f7 100644 --- a/src/bos/server/controllers/v2/healthz.py +++ b/src/bos/server/controllers/v2/healthz.py @@ -29,7 +29,7 @@ DB = redis_db_utils.get_wrapper(db='options') -LOGGER = logging.getLogger('bos.server.controllers.v2.healthz') +LOGGER = logging.getLogger(__name__) def _get_db_status(): diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index de52b024..e04774e3 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -33,7 +33,7 @@ from bos.server.models.v2_options import V2Options as Options from bos.server.utils import get_request_json -LOGGER = logging.getLogger('bos.server.controllers.v2.options') +LOGGER = logging.getLogger(__name__) DB = dbutils.get_wrapper(db='options') # We store all options as json under this key so that the data format is # similar to other data stored in the database, and to make retrieval of all @@ -48,6 +48,7 @@ class OptionsData(OptionsCache): This caches the options so that frequent use of these options do not all result in DB calls. """ + def _get_options(self) -> dict: """Retrieves the current options from the BOS DB""" LOGGER.debug("Retrieving options data from BOS DB") @@ -60,7 +61,8 @@ def _get_options(self) -> dict: def _init(): # Start log level updater - log_level_updater = threading.Thread(target=check_v2_logging_level, args=()) + log_level_updater = threading.Thread(target=check_v2_logging_level, + args=()) log_level_updater.start() # Cleanup old options @@ -69,7 +71,8 @@ def _init(): data = DB.get(OPTIONS_KEY) break except Exception as err: - LOGGER.info('Database is not yet available (%s)', exc_type_msg(err)) + LOGGER.info('Database is not yet available (%s)', + exc_type_msg(err)) time.sleep(1) if not data: return @@ -131,9 +134,9 @@ def patch_v2_options(): data = get_request_json() except Exception as err: LOGGER.error("Error parsing PATCH request data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) if OPTIONS_KEY not in DB: DB.put(OPTIONS_KEY, {}) @@ -159,5 +162,6 @@ def check_v2_logging_level(): if 'logging_level' in data: update_log_level(data['logging_level']) except Exception as err: - LOGGER.debug("Error checking or updating log level: %s", exc_type_msg(err)) + LOGGER.debug("Error checking or updating log level: %s", + exc_type_msg(err)) time.sleep(5) diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index 9a5aaa0e..2ce292ed 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -43,14 +43,14 @@ from bos.server.models.v2_session_create import V2SessionCreate as SessionCreate # noqa: E501 from bos.server.utils import get_request_json, ParsingException - -LOGGER = logging.getLogger('bos.server.controllers.v2.session') +LOGGER = logging.getLogger(__name__) DB = dbutils.get_wrapper(db='sessions') COMPONENTS_DB = dbutils.get_wrapper(db='components') STATUS_DB = dbutils.get_wrapper(db='session_status') MAX_COMPONENTS_IN_ERROR_DETAILS = 10 LIMIT_NID_RE = re.compile(r'^[&!]*nid') + @reject_invalid_tenant @dbutils.redis_error_handler def post_v2_session(): # noqa: E501 @@ -64,12 +64,13 @@ def post_v2_session(): # noqa: E501 LOGGER.debug("POST /v2/sessions invoked post_v2_session") # -- Validation -- try: - session_create = SessionCreate.from_dict(get_request_json()) # noqa: E501 + session_create = SessionCreate.from_dict( + get_request_json()) # noqa: E501 except Exception as err: LOGGER.error("Error parsing POST request data: %s", exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) options_data = OptionsData() @@ -80,8 +81,9 @@ def post_v2_session(): # noqa: E501 return msg, 400 # If a limit is specified, check it for nids - if session_create.limit and any(LIMIT_NID_RE.match(limit_item) - for limit_item in session_create.limit.split(',')): + if session_create.limit and any( + LIMIT_NID_RE.match(limit_item) + for limit_item in session_create.limit.split(',')): msg = f"session limit appears to contain NIDs: {session_create.limit}" if options_data.reject_nids: msg = f"reject_nids: {msg}" @@ -93,7 +95,8 @@ def post_v2_session(): # noqa: E501 LOGGER.warning(msg) template_name = session_create.template_name - LOGGER.debug("Template Name: %s operation: %s", template_name, session_create.operation) + LOGGER.debug("Template Name: %s operation: %s", template_name, + session_create.operation) # Check that the template_name exists. session_template_response = get_v2_sessiontemplate(template_name) if isinstance(session_template_response, ConnexionResponse): @@ -103,7 +106,9 @@ def post_v2_session(): # noqa: E501 session_template, _ = session_template_response # Validate health/validity of the sessiontemplate before creating a session - error_code, msg = validate_boot_sets(session_template, session_create.operation, template_name, + error_code, msg = validate_boot_sets(session_template, + session_create.operation, + template_name, options_data=options_data) if error_code >= BootSetStatus.ERROR: LOGGER.error("Session template fails check: %s", msg) @@ -112,14 +117,13 @@ def post_v2_session(): # noqa: E501 # -- Setup Record -- tenant = get_tenant_from_header() session = _create_session(session_create, tenant) - session_key = get_tenant_aware_key(session.name, tenant) + session_key = get_tenant_aware_key(session.name, tenant) if session_key in DB: LOGGER.warning("v2 session named %s already exists", session.name) return connexion.problem( detail=f"A session with the name {session.name} already exists", status=409, - title="Conflicting session name" - ) + title="Conflicting session name") session_data = session.to_dict() response = DB.put(session_key, session_data) return response, 201 @@ -158,16 +162,18 @@ def patch_v2_session(session_id): try: patch_data_json = get_request_json() except Exception as err: - LOGGER.error("Error parsing PATCH '%s' request data: %s", session_id, exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + LOGGER.error("Error parsing PATCH '%s' request data: %s", session_id, + exc_type_msg(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) session_key = get_tenant_aware_key(session_id, get_tenant_from_header()) if session_key not in DB: LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( - status=404, title="Session could not found.", + status=404, + title="Session could not found.", detail=f"Session {session_id} could not be found") component = DB.patch(session_key, patch_data_json) @@ -188,7 +194,8 @@ def get_v2_session(session_id): # noqa: E501 if session_key not in DB: LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( - status=404, title="Session could not found.", + status=404, + title="Session could not found.", detail=f"Session {session_id} could not be found") session = DB.get(session_key) return session, 200 @@ -200,10 +207,12 @@ def get_v2_sessions(min_age=None, max_age=None, status=None): # noqa: E501 List all sessions """ - LOGGER.debug("GET /v2/sessions invoked get_v2_sessions with min_age=%s max_age=%s status=%s", - min_age, max_age, status) + LOGGER.debug( + "GET /v2/sessions invoked get_v2_sessions with min_age=%s max_age=%s status=%s", + min_age, max_age, status) response = _get_filtered_sessions(tenant=get_tenant_from_header(), - min_age=min_age, max_age=max_age, + min_age=min_age, + max_age=max_age, status=status) LOGGER.debug("get_v2_sessions returning %d sessions", len(response)) return response, 200 @@ -215,12 +224,14 @@ def delete_v2_session(session_id): # noqa: E501 Delete the session by session id """ - LOGGER.debug("DELETE /v2/sessions/%s invoked delete_v2_session", session_id) + LOGGER.debug("DELETE /v2/sessions/%s invoked delete_v2_session", + session_id) session_key = get_tenant_aware_key(session_id, get_tenant_from_header()) if session_key not in DB: LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( - status=404, title="Session could not found.", + status=404, + title="Session could not found.", detail=f"Session {session_id} could not be found") if session_key in STATUS_DB: STATUS_DB.delete(session_key) @@ -230,19 +241,19 @@ def delete_v2_session(session_id): # noqa: E501 @dbutils.redis_error_handler def delete_v2_sessions(min_age=None, max_age=None, status=None): # noqa: E501 LOGGER.debug( - "DELETE /v2/sessions invoked delete_v2_sessions with min_age=%s max_age=%s status=%s", - min_age, max_age, status) + "DELETE /v2/sessions invoked delete_v2_sessions with min_age=%s max_age=%s status=%s", + min_age, max_age, status) tenant = get_tenant_from_header() try: - sessions = _get_filtered_sessions(tenant=tenant, min_age=min_age, max_age=max_age, + sessions = _get_filtered_sessions(tenant=tenant, + min_age=min_age, + max_age=max_age, status=status) except ParsingException as err: LOGGER.error("Error parsing age field: %s", exc_type_msg(err)) - return connexion.problem( - detail=str(err), - status=400, - title='Error parsing age field' - ) + return connexion.problem(detail=str(err), + status=400, + title='Error parsing age field') for session in sessions: session_key = get_tenant_aware_key(session['name'], tenant) @@ -262,15 +273,19 @@ def get_v2_session_status(session_id): # noqa: E501 Return: Session Status Dictionary, Status Code """ - LOGGER.debug("GET /v2/sessions/status/%s invoked get_v2_session_status", session_id) + LOGGER.debug("GET /v2/sessions/status/%s invoked get_v2_session_status", + session_id) session_key = get_tenant_aware_key(session_id, get_tenant_from_header()) if session_key not in DB: LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( - status=404, title="Session could not found.", + status=404, + title="Session could not found.", detail=f"Session {session_id} could not be found") session = DB.get(session_key) - if session.get("status", {}).get("status") == "complete" and session_key in STATUS_DB: + if session.get( + "status", + {}).get("status") == "complete" and session_key in STATUS_DB: # If the session is complete and the status is saved, # return the status from completion time return STATUS_DB.get(session_key), 200 @@ -286,12 +301,14 @@ def save_v2_session_status(session_id): # noqa: E501 Return: Session Status Dictionary, Status Code """ - LOGGER.debug("POST /v2/sessions/status/%s invoked save_v2_session_status", session_id) + LOGGER.debug("POST /v2/sessions/status/%s invoked save_v2_session_status", + session_id) session_key = get_tenant_aware_key(session_id, get_tenant_from_header()) if session_key not in DB: LOGGER.warning("Could not find v2 session %s", session_id) return connexion.problem( - status=404, title="Session could not found.", + status=404, + title="Session could not found.", detail=f"Session {session_id} could not be found") return STATUS_DB.put(session_key, _get_v2_session_status(session_key)), 200 @@ -313,8 +330,10 @@ def _get_filtered_sessions(tenant, min_age, max_age, status): LOGGER.warning('Unable to parse max_age: %s', max_age) raise ParsingException(e) from e if any([min_start, max_start, status, tenant]): - response = [r for r in response if _matches_filter(r, tenant, min_start, max_start, - status)] + response = [ + r for r in response + if _matches_filter(r, tenant, min_start, max_start, status) + ] return response @@ -341,34 +360,46 @@ def _get_v2_session_status(session_key, session=None): session_id = session.get("name", {}) tenant_id = session.get("tenant") components = get_v2_components_data(session=session_id, tenant=tenant_id) - staged_components = get_v2_components_data(staged_session=session_id, tenant=tenant_id) + staged_components = get_v2_components_data(staged_session=session_id, + tenant=tenant_id) num_managed_components = len(components) + len(staged_components) if num_managed_components: component_phase_counts = Counter([ - c.get('status', {}).get('phase') - for c in components - if (c.get('enabled') and - c.get('status').get('status_override') != Status.on_hold)]) + c.get('status', {}).get('phase') for c in components + if (c.get('enabled') + and c.get('status').get('status_override') != Status.on_hold) + ]) component_phase_counts['successful'] = len([ - c for c in components if c.get('status',{}).get('status') == Status.stable]) + c for c in components + if c.get('status', {}).get('status') == Status.stable + ]) component_phase_counts['failed'] = len([ - c for c in components if c.get('status', {}).get('status') == Status.failed]) + c for c in components + if c.get('status', {}).get('status') == Status.failed + ]) component_phase_counts['staged'] = len(staged_components) component_phase_percents = { - phase: (component_phase_counts[phase] / num_managed_components) * 100 - for phase in component_phase_counts} + phase: + (component_phase_counts[phase] / num_managed_components) * 100 + for phase in component_phase_counts + } else: component_phase_percents = {} component_errors_data = defaultdict(set) for component in components: if component.get('error'): - component_errors_data[component.get('error')].add(component.get('id')) + component_errors_data[component.get('error')].add( + component.get('id')) component_errors = {} for error, components in component_errors_data.items(): - component_list = ','.join(list(components)[:MAX_COMPONENTS_IN_ERROR_DETAILS]) + component_list = ','.join( + list(components)[:MAX_COMPONENTS_IN_ERROR_DETAILS]) if len(components) > MAX_COMPONENTS_IN_ERROR_DETAILS: component_list += '...' - component_errors[error] = {'count': len(components), 'list': component_list} + component_errors[error] = { + 'count': len(components), + 'list': component_list + } session_status = session.get('status', {}) start_time = session_status.get('start_time') end_time = session_status.get('end_time') @@ -377,20 +408,30 @@ def _get_v2_session_status(session_key, session=None): else: duration = str(get_current_time() - load_timestamp(start_time)) status = { - 'status': session_status.get('status', ''), - 'managed_components_count': num_managed_components, + 'status': + session_status.get('status', ''), + 'managed_components_count': + num_managed_components, 'phases': { - 'percent_complete': round( - component_phase_percents.get('successful', - 0) + component_phase_percents.get('failed', 0), 2), - 'percent_powering_on': round(component_phase_percents.get(Phase.powering_on, 0), 2), - 'percent_powering_off': round(component_phase_percents.get(Phase.powering_off, 0), 2), - 'percent_configuring': round(component_phase_percents.get(Phase.configuring, 0), 2), + 'percent_complete': + round( + component_phase_percents.get('successful', 0) + + component_phase_percents.get('failed', 0), 2), + 'percent_powering_on': + round(component_phase_percents.get(Phase.powering_on, 0), 2), + 'percent_powering_off': + round(component_phase_percents.get(Phase.powering_off, 0), 2), + 'percent_configuring': + round(component_phase_percents.get(Phase.configuring, 0), 2), }, - 'percent_staged': round(component_phase_percents.get('staged', 0), 2), - 'percent_successful': round(component_phase_percents.get('successful', 0), 2), - 'percent_failed': round(component_phase_percents.get('failed', 0), 2), - 'error_summary': component_errors, + 'percent_staged': + round(component_phase_percents.get('staged', 0), 2), + 'percent_successful': + round(component_phase_percents.get('successful', 0), 2), + 'percent_failed': + round(component_phase_percents.get('failed', 0), 2), + 'error_summary': + component_errors, 'timing': { 'start_time': start_time, 'end_time': end_time, diff --git a/src/bos/server/controllers/v2/sessiontemplates.py b/src/bos/server/controllers/v2/sessiontemplates.py index b10eff4d..ac80c52b 100644 --- a/src/bos/server/controllers/v2/sessiontemplates.py +++ b/src/bos/server/controllers/v2/sessiontemplates.py @@ -32,27 +32,33 @@ from bos.server.utils import get_request_json from .boot_set import validate_boot_sets, validate_sanitize_boot_sets -LOGGER = logging.getLogger('bos.server.controllers.v2.sessiontemplates') +LOGGER = logging.getLogger(__name__) DB = dbutils.get_wrapper(db='session_templates') EXAMPLE_BOOT_SET = { "type": "s3", "etag": "boot-image-s3-etag", "kernel_parameters": "your-kernel-parameters", - "cfs": {"configuration": "bootset-specific-cfs-override"}, - "node_list": [ - "xname1", "xname2", "xname3"], + "cfs": { + "configuration": "bootset-specific-cfs-override" + }, + "node_list": ["xname1", "xname2", "xname3"], "path": "s3://boot-images/boot-image-ims-id/manifest.json", "rootfs_provider": "cpss3", - "rootfs_provider_passthrough": "dvs:api-gw-service-nmn.local:300:hsn0,nmn0:0"} + "rootfs_provider_passthrough": + "dvs:api-gw-service-nmn.local:300:hsn0,nmn0:0" +} EXAMPLE_SESSION_TEMPLATE = { "boot_sets": { - "name_your_boot_set": EXAMPLE_BOOT_SET}, + "name_your_boot_set": EXAMPLE_BOOT_SET + }, "cfs": { - "configuration": "default-sessiontemplate-cfs-config"}, + "configuration": "default-sessiontemplate-cfs-config" + }, "enable_cfs": True, - "name": "name-your-template"} + "name": "name-your-template" +} @reject_invalid_tenant @@ -62,24 +68,26 @@ def put_v2_sessiontemplate(session_template_id): # noqa: E501 Creates a new session template. # noqa: E501 """ - LOGGER.debug("PUT /v2/sessiontemplates/%s invoked put_v2_sessiontemplate", session_template_id) + LOGGER.debug("PUT /v2/sessiontemplates/%s invoked put_v2_sessiontemplate", + session_template_id) try: template_data = get_request_json() except Exception as err: - LOGGER.error("Error parsing PUT '%s' request data: %s", session_template_id, - exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + LOGGER.error("Error parsing PUT '%s' request data: %s", + session_template_id, exc_type_msg(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) try: validate_sanitize_session_template(session_template_id, template_data) except Exception as err: - LOGGER.error("Error creating session template '%s': %s", session_template_id, - exc_type_msg(err)) + LOGGER.error("Error creating session template '%s': %s", + session_template_id, exc_type_msg(err)) LOGGER.debug("Full template: %s", template_data) return connexion.problem( - status=400, title="The session template could not be created.", + status=400, + title="The session template could not be created.", detail=str(err)) tenant = get_tenant_from_header() @@ -97,7 +105,8 @@ def get_v2_sessiontemplates(): # noqa: E501 """ LOGGER.debug("GET /v2/sessiontemplates invoked get_v2_sessiontemplates") response = _get_filtered_templates(tenant=get_tenant_from_header()) - LOGGER.debug("get_v2_sessiontemplates returning %d templates", len(response)) + LOGGER.debug("get_v2_sessiontemplates returning %d templates", + len(response)) return response, 200 @@ -108,12 +117,15 @@ def get_v2_sessiontemplate(session_template_id): Get the session template by session template ID """ - LOGGER.debug("GET /v2/sessiontemplates/%s invoked get_v2_sessiontemplate", session_template_id) - template_key = get_tenant_aware_key(session_template_id, get_tenant_from_header()) + LOGGER.debug("GET /v2/sessiontemplates/%s invoked get_v2_sessiontemplate", + session_template_id) + template_key = get_tenant_aware_key(session_template_id, + get_tenant_from_header()) if template_key not in DB: LOGGER.warning("Session template not found: %s", session_template_id) return connexion.problem( - status=404, title="Sessiontemplate could not found.", + status=404, + title="Sessiontemplate could not found.", detail=f"Sessiontemplate {session_template_id} could not be found") template = DB.get(template_key) return template, 200 @@ -126,7 +138,9 @@ def get_v2_sessiontemplatetemplate(): Get the example session template """ - LOGGER.debug("GET /v2/sessiontemplatetemplate invoked get_v2_sessiontemplatetemplate") + LOGGER.debug( + "GET /v2/sessiontemplatetemplate invoked get_v2_sessiontemplatetemplate" + ) return EXAMPLE_SESSION_TEMPLATE, 200 @@ -137,13 +151,16 @@ def delete_v2_sessiontemplate(session_template_id): Delete the session template by session template ID """ - LOGGER.debug("DELETE /v2/sessiontemplates/%s invoked delete_v2_sessiontemplate", - session_template_id) - template_key = get_tenant_aware_key(session_template_id, get_tenant_from_header()) + LOGGER.debug( + "DELETE /v2/sessiontemplates/%s invoked delete_v2_sessiontemplate", + session_template_id) + template_key = get_tenant_aware_key(session_template_id, + get_tenant_from_header()) if template_key not in DB: LOGGER.warning("Session template not found: %s", session_template_id) return connexion.problem( - status=404, title="Sessiontemplate could not found.", + status=404, + title="Sessiontemplate could not found.", detail=f"Sessiontemplate {session_template_id} could not be found") return DB.delete(template_key), 204 @@ -155,31 +172,35 @@ def patch_v2_sessiontemplate(session_template_id): Patch the session template by session template ID """ - LOGGER.debug("PATCH /v2/sessiontemplates/%s invoked patch_v2_sessiontemplate", - session_template_id) - template_key = get_tenant_aware_key(session_template_id, get_tenant_from_header()) + LOGGER.debug( + "PATCH /v2/sessiontemplates/%s invoked patch_v2_sessiontemplate", + session_template_id) + template_key = get_tenant_aware_key(session_template_id, + get_tenant_from_header()) if template_key not in DB: LOGGER.warning("Session template not found: %s", session_template_id) return connexion.problem( - status=404, title="Sessiontemplate could not found.", + status=404, + title="Sessiontemplate could not found.", detail=f"Sessiontemplate {session_template_id} could not be found") try: template_data = get_request_json() except Exception as err: - LOGGER.error("Error parsing PATCH '%s' request data: %s", session_template_id, - exc_type_msg(err)) - return connexion.problem( - status=400, title="Error parsing the data provided.", - detail=str(err)) + LOGGER.error("Error parsing PATCH '%s' request data: %s", + session_template_id, exc_type_msg(err)) + return connexion.problem(status=400, + title="Error parsing the data provided.", + detail=str(err)) try: validate_sanitize_session_template(session_template_id, template_data) except Exception as err: - LOGGER.error("Error patching session template '%s': %s", session_template_id, - exc_type_msg(err)) + LOGGER.error("Error patching session template '%s': %s", + session_template_id, exc_type_msg(err)) return connexion.problem( - status=400, title="The session template could not be patched.", + status=400, + title="The session template could not be patched.", detail=str(err)) return DB.patch(template_key, template_data), 200 @@ -191,8 +212,9 @@ def validate_v2_sessiontemplate(session_template_id: str): Validate a V2 session template. Look for missing elements or errors that would prevent a session from being launched using this template. """ - LOGGER.debug("GET /v2/sessiontemplatesvalid/%s invoked validate_v2_sessiontemplate", - session_template_id) + LOGGER.debug( + "GET /v2/sessiontemplatesvalid/%s invoked validate_v2_sessiontemplate", + session_template_id) data, status_code = get_v2_sessiontemplate(session_template_id) if status_code != 200: diff --git a/src/bos/server/dbs/boot_artifacts.py b/src/bos/server/dbs/boot_artifacts.py index 80f02aad..10f68bec 100644 --- a/src/bos/server/dbs/boot_artifacts.py +++ b/src/bos/server/dbs/boot_artifacts.py @@ -26,7 +26,7 @@ from bos.common.utils import get_current_timestamp from bos.server import redis_db_utils as dbutils -LOGGER = logging.getLogger('bos.server.dbs.boot_artifacts') +LOGGER = logging.getLogger(__name__) TOKENS_DB = dbutils.get_wrapper(db='bss_tokens_boot_artifacts') @@ -40,20 +40,23 @@ class BssTokenUnknown(BssTokenException): """ -def record_boot_artifacts(token: str, - kernel: str, - kernel_parameters: str, +def record_boot_artifacts(token: str, kernel: str, kernel_parameters: str, initrd: str): """ Associate the BSS token with the boot artifacts. BSS returns a token after BOS asks it to create or update the boot artifacts. """ - LOGGER.info("Logging BSS token and boot artifacts: token='%s' kernel='%s' " - "kernel_parameters='%s' initrd='%s'", token, kernel, kernel_parameters, initrd) - TOKENS_DB.put(token, {"kernel": kernel, - "kernel_parameters": kernel_parameters, - "initrd": initrd, - "timestamp": get_current_timestamp()}) + LOGGER.info( + "Logging BSS token and boot artifacts: token='%s' kernel='%s' " + "kernel_parameters='%s' initrd='%s'", token, kernel, kernel_parameters, + initrd) + TOKENS_DB.put( + token, { + "kernel": kernel, + "kernel_parameters": kernel_parameters, + "initrd": initrd, + "timestamp": get_current_timestamp() + }) def get_boot_artifacts(token: str) -> dict: diff --git a/src/bos/server/migrations/__main__.py b/src/bos/server/migrations/__main__.py index 891fdea1..571bd131 100644 --- a/src/bos/server/migrations/__main__.py +++ b/src/bos/server/migrations/__main__.py @@ -22,7 +22,6 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # - """ Starting in CSM 1.6, BOS is enforcing many API restrictions for the first time. When migrating to this BOS version, this tool will attempt to clean up the BOS @@ -57,8 +56,7 @@ from .db import COMP_DB, SESS_DB, TEMP_DB from .sanitize import sanitize_component, sanitize_session, sanitize_session_template - -LOGGER = logging.getLogger('bos.server.migration') +LOGGER = logging.getLogger(__name__) def main(): diff --git a/src/bos/server/migrations/db.py b/src/bos/server/migrations/db.py index 4824e6cc..ef863243 100644 --- a/src/bos/server/migrations/db.py +++ b/src/bos/server/migrations/db.py @@ -26,35 +26,39 @@ import bos.server.redis_db_utils as dbutils -LOGGER = logging.getLogger('bos.server.migration') +LOGGER = logging.getLogger(__name__) -TEMP_DB=dbutils.get_wrapper(db='session_templates') -SESS_DB=dbutils.get_wrapper(db='sessions') -STAT_DB=dbutils.get_wrapper(db='session_status') -COMP_DB=dbutils.get_wrapper(db='components') +TEMP_DB = dbutils.get_wrapper(db='session_templates') +SESS_DB = dbutils.get_wrapper(db='sessions') +STAT_DB = dbutils.get_wrapper(db='session_status') +COMP_DB = dbutils.get_wrapper(db='components') -def delete_from_db(db: dbutils.DBWrapper, key: str, err_msg: str|None=None) -> None: +def delete_from_db(db: dbutils.DBWrapper, + key: str, + err_msg: str | None = None) -> None: if err_msg is None: LOGGER.warning("Deleting %s under DB key '%s'", db.db_string, key) else: - LOGGER.error("%s; Deleting %s under DB key '%s'", err_msg, db.db_string, key) + LOGGER.error("%s; Deleting %s under DB key '%s'", err_msg, + db.db_string, key) data = db.get_and_delete(key) if data: LOGGER.info("Deleted %s '%s': %s", db.db_string, key, data) else: - LOGGER.warning("Could not delete %s '%s' -- does not exist", db.db_string, key) + LOGGER.warning("Could not delete %s '%s' -- does not exist", + db.db_string, key) -def delete_component(key: str, err_msg: str|None=None) -> None: +def delete_component(key: str, err_msg: str | None = None) -> None: delete_from_db(COMP_DB, key, err_msg) -def delete_template(key: str, err_msg: str|None=None) -> None: +def delete_template(key: str, err_msg: str | None = None) -> None: delete_from_db(TEMP_DB, key, err_msg) -def delete_session(key: str, err_msg: str|None=None) -> None: +def delete_session(key: str, err_msg: str | None = None) -> None: delete_from_db(SESS_DB, key, err_msg) LOGGER.info("Deleting associated session status, if it exists") delete_from_db(STAT_DB, key, err_msg) diff --git a/src/bos/server/migrations/sanitize.py b/src/bos/server/migrations/sanitize.py index d4171686..946b0505 100644 --- a/src/bos/server/migrations/sanitize.py +++ b/src/bos/server/migrations/sanitize.py @@ -36,14 +36,13 @@ get_required_field, get_validate_tenant, is_valid_available_template_name, \ validate_bootset_path, validate_against_schema - -LOGGER = logging.getLogger('bos.server.migration') +LOGGER = logging.getLogger(__name__) ALPHANUMERIC = string.ascii_letters + string.digits TEMPLATE_NAME_CHARACTERS = ALPHANUMERIC + '-._' -def sanitize_component(key: str|bytes, data: dict) -> None: +def sanitize_component(key: str | bytes, data: dict) -> None: """ If the id field is missing or invalid, delete the component """ @@ -53,7 +52,7 @@ def sanitize_component(key: str|bytes, data: dict) -> None: delete_component(key, str(exc)) -def sanitize_session(key: str|bytes, data: dict) -> None: +def sanitize_session(key: str | bytes, data: dict) -> None: """ If the name field is missing, or if the name or tenant fields are invalid, delete the session. """ @@ -63,7 +62,7 @@ def sanitize_session(key: str|bytes, data: dict) -> None: delete_session(key, str(exc)) -def sanitize_session_template(key: str|bytes, data: dict) -> None: +def sanitize_session_template(key: str | bytes, data: dict) -> None: """ Session templates are the things most likely to run afoul of the API spec. This attempts to automatically fix them if at all possible, only deleting them @@ -75,7 +74,7 @@ def sanitize_session_template(key: str|bytes, data: dict) -> None: delete_template(key, str(exc)) -def _sanitize_session_template(key: str|bytes, data: dict) -> None: +def _sanitize_session_template(key: str | bytes, data: dict) -> None: """ Validates and tries to sanitize the session template. If there are correctable errors, the function will update the database @@ -97,8 +96,10 @@ def _sanitize_session_template(key: str|bytes, data: dict) -> None: # Make a copy of the session template. If we identify problems, we will see if we can correct # them in the copy. While copying, remove any fields that are no longer in the spec - new_data = { k: copy.deepcopy(v) for k,v in data.items() - if k in validator.session_template_fields } + new_data = { + k: copy.deepcopy(v) + for k, v in data.items() if k in validator.session_template_fields + } # Check and sanitize each boot set for bsname, bsdata in new_data["boot_sets"].items(): @@ -120,7 +121,8 @@ def _sanitize_session_template(key: str|bytes, data: dict) -> None: return # This means the data changed, so we need to update the entry under the existing key - LOGGER.warning("Updating session template to comply with the BOS API schema") + LOGGER.warning( + "Updating session template to comply with the BOS API schema") LOGGER.warning("Old template data: %s", data) LOGGER.warning("New template data: %s", new_data) TEMP_DB.put(key, new_data) @@ -131,7 +133,8 @@ def _sanitize_session_template(key: str|bytes, data: dict) -> None: if data == new_data: LOGGER.warning(base_msg) else: - LOGGER.warning("%s and updating it to comply with the BOS API schema", base_msg) + LOGGER.warning("%s and updating it to comply with the BOS API schema", + base_msg) delete_template(key, data) @@ -146,7 +149,9 @@ def _sanitize_session_template(key: str|bytes, data: dict) -> None: try: validate_against_schema(new_data, "V2SessionTemplate") except ValidationError: - LOGGER.error("New session template does not follow schema -- it will not be saved") + LOGGER.error( + "New session template does not follow schema -- it will not be saved" + ) return TEMP_DB.put(new_key, new_data) @@ -170,7 +175,8 @@ def sanitize_description_field(data: dict) -> None: # Log a warning and delete it if it is not a string if not isinstance(description, str): - LOGGER.warning("Removing non-string 'description' field from session template") + LOGGER.warning( + "Removing non-string 'description' field from session template") del data["description"] return @@ -179,7 +185,7 @@ def sanitize_description_field(data: dict) -> None: data["description"] = description[:1023] -def sanitize_bootset(bsname: str, bsdata: dict) -> str|None: +def sanitize_bootset(bsname: str, bsdata: dict) -> str | None: """ Corrects in-place bsdata. Returns an error message if this proves impossible. @@ -201,7 +207,9 @@ def sanitize_bootset(bsname: str, bsdata: dict) -> str|None: bsdata["arch"] = DEFAULT_ARCH # Remove any fields that are no longer in the spec - bad_fields = [ field for field in bsdata if field not in validator.boot_set_fields ] + bad_fields = [ + field for field in bsdata if field not in validator.boot_set_fields + ] for field in bad_fields: del bsdata[field] @@ -216,7 +224,7 @@ def sanitize_bootset(bsname: str, bsdata: dict) -> str|None: # Use list() since we will be modifying the dict while iterating over its contents for field, value in list(bsdata.items()): # We have already dealt with 'cfs', 'path', and 'type', so we can skip those - if field in { 'cfs', 'path', 'type' }: + if field in {'cfs', 'path', 'type'}: continue # Delete None-valued fields that are not nullable (No boot set fields are nullable) @@ -243,7 +251,8 @@ def sanitize_bootset(bsname: str, bsdata: dict) -> str|None: return raise ValidationError( - f"Boot set '{bsname}' has no non-empty node fields ({HARDWARE_SPECIFIER_FIELDS})") + f"Boot set '{bsname}' has no non-empty node fields ({HARDWARE_SPECIFIER_FIELDS})" + ) def sanitize_cfs_field(data: dict) -> None: @@ -272,7 +281,7 @@ def sanitize_cfs_field(data: dict) -> None: raise ValidationError("'cfs' field value has invalid type") # Remove any fields that are no longer in the spec - bad_fields = [ field for field in cfs if field not in validator.cfs_fields ] + bad_fields = [field for field in cfs if field not in validator.cfs_fields] for field in bad_fields: del cfs[field] @@ -287,7 +296,7 @@ def sanitize_cfs_field(data: dict) -> None: del data["cfs"] -def get_unused_legal_template_name(name: str, tenant: str|None) -> str: +def get_unused_legal_template_name(name: str, tenant: str | None) -> str: """ If the current name is legal, return it unchanged. Otherwise, try to find a name which is not in use and which is legal per the spec. @@ -302,12 +311,13 @@ def get_unused_legal_template_name(name: str, tenant: str|None) -> str: if not name or not any(c in TEMPLATE_NAME_CHARACTERS for c in name): raise - LOGGER.warning("Session template name '%s' (tenant: %s) does not follow schema. " - "Will attempt to rename to a legal name", name, tenant) + LOGGER.warning( + "Session template name '%s' (tenant: %s) does not follow schema. " + "Will attempt to rename to a legal name", name, tenant) # Strip out illegal characters, but replace spaces with underscores and prepend 'auto_renamed_' - new_name_base = 'auto_renamed_' + ''.join([ c for c in name.replace(' ','_') - if c in TEMPLATE_NAME_CHARACTERS ]) + new_name_base = 'auto_renamed_' + ''.join( + [c for c in name.replace(' ', '_') if c in TEMPLATE_NAME_CHARACTERS]) # Trim to 127 characters, if it exceeds that new_name = new_name_base[:127] @@ -321,13 +331,15 @@ def get_unused_legal_template_name(name: str, tenant: str|None) -> str: # Trying all 2 character alphanumeric suffixes gives 1953 options, which is enough of an effort # for us to make here. for suffix_length in range(1, 3): - for suffix in itertools.combinations_with_replacement(ALPHANUMERIC, suffix_length): + for suffix in itertools.combinations_with_replacement( + ALPHANUMERIC, suffix_length): new_name = f'{new_name_base[:126-suffix_length]}_{suffix}' if is_valid_available_template_name(new_name, tenant): return new_name - LOGGER.error("Unable to find unused valid new name for session template '%s' (tenant: %s)", - name, tenant) + LOGGER.error( + "Unable to find unused valid new name for session template '%s' (tenant: %s)", + name, tenant) raise ValidationError("Name does not follow schema") @@ -337,10 +349,9 @@ def log_rename_in_template_description(old_name: str, data: dict) -> None: template. Failing that, if possible, at least record that it was renamed. """ rename_messages = [ - f"Former name: {old_name}", - "Renamed during BOS upgrade", - "Auto-renamed", - "Renamed" ] + f"Former name: {old_name}", "Renamed during BOS upgrade", + "Auto-renamed", "Renamed" + ] current_description = data.get("description", "") for msg in rename_messages: diff --git a/src/bos/server/migrations/validate.py b/src/bos/server/migrations/validate.py index d4a4625e..bb1ccc2c 100644 --- a/src/bos/server/migrations/validate.py +++ b/src/bos/server/migrations/validate.py @@ -31,8 +31,7 @@ from .db import TEMP_DB - -LOGGER = logging.getLogger('bos.server.migration') +LOGGER = logging.getLogger(__name__) class ValidationError(Exception): @@ -41,7 +40,7 @@ class ValidationError(Exception): """ -def check_session(key: str|bytes, data: dict) -> None: +def check_session(key: str | bytes, data: dict) -> None: """ Raises a ValidationError if the data contains fatal errors. """ @@ -52,7 +51,7 @@ def check_session(key: str|bytes, data: dict) -> None: check_keys(key, expected_db_key) -def check_component(key: str|bytes, data: dict) -> None: +def check_component(key: str | bytes, data: dict) -> None: """ Raises a ValidationError if the data contains fatal errors. """ @@ -61,7 +60,7 @@ def check_component(key: str|bytes, data: dict) -> None: check_keys(key, compid) -def get_validate_tenant(data: dict) -> str|None: +def get_validate_tenant(data: dict) -> str | None: """ If no tenant field present, return None. If the tenant field value is valid, return it. @@ -81,10 +80,11 @@ def validate_bootset_path(bsname: str, bsdata: dict) -> None: try: validate_against_schema(path, "BootManifestPath") except ValidationError as exc: - raise ValidationError(f"Boot set '{bsname}' has invalid 'path' field: {exc}") from exc + raise ValidationError( + f"Boot set '{bsname}' has invalid 'path' field: {exc}") from exc -def check_keys(actual: str|bytes, expected: str|bytes) -> str|None: +def check_keys(actual: str | bytes, expected: str | bytes) -> str | None: """ Converts both keys to strings. Raises ValidationError if the strings do not match @@ -95,10 +95,11 @@ def check_keys(actual: str|bytes, expected: str|bytes) -> str|None: expected = expected.decode() if actual != expected: raise ValidationError( - f"Actual DB key ('{actual}') does not match expected key ('{expected}')") + f"Actual DB key ('{actual}') does not match expected key ('{expected}')" + ) -def is_valid_available_template_name(name: str, tenant: str|None) -> bool: +def is_valid_available_template_name(name: str, tenant: str | None) -> bool: if get_tenant_aware_key(name, tenant) in TEMP_DB: return False try: @@ -116,7 +117,8 @@ def validate_against_schema(obj: Any, schema_name: str) -> None: validator.validate(obj, schema_name) except Exception as exc: LOGGER.error(exc_type_msg(exc)) - raise ValidationError(f"Does not follow {schema_name} schema: {obj}") from exc + raise ValidationError( + f"Does not follow {schema_name} schema: {obj}") from exc def get_required_field(field: str, data: dict) -> Any: diff --git a/src/bos/server/redis_db_utils.py b/src/bos/server/redis_db_utils.py index 988970fb..9a4921b6 100644 --- a/src/bos/server/redis_db_utils.py +++ b/src/bos/server/redis_db_utils.py @@ -21,9 +21,11 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +from itertools import batched import functools import json import logging +from typing import Callable, Optional import connexion import redis @@ -31,8 +33,10 @@ from bos.common.utils import exc_type_msg LOGGER = logging.getLogger(__name__) -DATABASES = ["options", "components", "session_templates", "sessions", "bss_tokens_boot_artifacts", - "session_status"] # Index is the db id. +DATABASES = [ + "options", "components", "session_templates", "sessions", + "bss_tokens_boot_artifacts", "session_status" +] # Index is the db id. DB_HOST = 'cray-bos-db' DB_PORT = 6379 @@ -64,13 +68,13 @@ def _get_db_id(self, db): def _get_client(self, db_id): """Create a connection with the database.""" try: - LOGGER.debug("Creating database connection" - "host: %s port: %s database: %s", - DB_HOST, DB_PORT, db_id) + LOGGER.debug( + "Creating database connection" + "host: %s port: %s database: %s", DB_HOST, DB_PORT, db_id) return redis.Redis(host=DB_HOST, port=DB_PORT, db=db_id) except Exception as err: - LOGGER.error("Failed to connect to database %s : %s", - db_id, exc_type_msg(err)) + LOGGER.error("Failed to connect to database %s : %s", db_id, + exc_type_msg(err)) raise @property @@ -104,6 +108,34 @@ def get_all(self): data.append(single_data) return data + def get_all_filtered(self, + filter_func: Callable[[dict], dict | None], + start_after_key: Optional[str] = None, + page_size: int = 0) -> list[dict]: + """ + Get an array of data for all keys after passing them through the specified filter + (discarding any for which the filter returns None) + If start_after_id is specified, all ids lexically <= that id will be skipped. + If page_size is specified, the list will be returned if it contains that many + elements, even if there may be more remaining. + """ + data = [] + for value in self.iter_values(start_after_key): + filtered_value = filter_func(value) + if filtered_value is not None: + data.append(filtered_value) + if page_size and len(data) == page_size: + break + return data + + def iter_values(self, start_after_key: Optional[str] = None): + all_keys = sorted({k.decode() for k in self.client.scan_iter()}) + if start_after_key is not None: + all_keys = [k for k in all_keys if k > start_after_key] + for next_keys in batched(all_keys, 500): + for datastr in self.client.mget(next_keys): + yield json.loads(datastr) if datastr else None + def get_all_as_dict(self): """Return a mapping from all keys to their corresponding data Based on https://github.com/redis/redis-py/issues/984#issuecomment-391404875 @@ -112,9 +144,11 @@ def get_all_as_dict(self): cursor = '0' while cursor != 0: cursor, keys = self.client.scan(cursor=cursor, count=1000) - values = [ json.loads(datastr) if datastr else None - for datastr in self.client.mget(keys) ] - keys = [ k.decode() for k in keys ] + values = [ + json.loads(datastr) if datastr else None + for datastr in self.client.mget(keys) + ] + keys = [k.decode() for k in keys] data.update(dict(zip(keys, values))) return data @@ -174,6 +208,7 @@ def info(self): def redis_error_handler(func): """Decorator for returning better errors if Redis is unreachable""" + @functools.wraps(func) def wrapper(*args, **kwargs): try: @@ -185,7 +220,8 @@ def wrapper(*args, **kwargs): except redis.exceptions.ConnectionError as e: LOGGER.error('Unable to connect to the Redis database: %s', e) return connexion.problem( - status=503, title='Unable to connect to the Redis database', + status=503, + title='Unable to connect to the Redis database', detail=str(e)) return wrapper diff --git a/src/bos/server/schema.py b/src/bos/server/schema.py index 3711d242..cc391041 100644 --- a/src/bos/server/schema.py +++ b/src/bos/server/schema.py @@ -27,13 +27,13 @@ import jsonschema - -LOGGER = logging.getLogger('bos.server.schema') +LOGGER = logging.getLogger(__name__) API_JSON_SCHEMA_PATH = "/app/lib/bos/server/openapi.jsonschema" class Validator: + def __init__(self): LOGGER.info("Loading API schema from %s", API_JSON_SCHEMA_PATH) with open(API_JSON_SCHEMA_PATH, "rt") as f: @@ -73,4 +73,5 @@ def boot_set_fields(self): def cfs_fields(self): return self.get_schema_fields("V2CfsParameters") + validator = Validator() diff --git a/src/bos/server/utils.py b/src/bos/server/utils.py index 1634f647..3bdd1328 100644 --- a/src/bos/server/utils.py +++ b/src/bos/server/utils.py @@ -26,7 +26,7 @@ import connexion -LOGGER = logging.getLogger('bos.server.utils') +LOGGER = logging.getLogger(__name__) class ParsingException(Exception): @@ -44,10 +44,11 @@ def canonize_xname(xname): :return: canonized xname :rtype: string """ - return re.sub(r'x0*(\d+)c0*(\d+)s0*(\d+)b0*(\d+)n0*(\d+)', r'x\1c\2s\3b\4n\5', xname.lower()) + return re.sub(r'x0*(\d+)c0*(\d+)s0*(\d+)b0*(\d+)n0*(\d+)', + r'x\1c\2s\3b\4n\5', xname.lower()) -def get_request_json(log_data = True): +def get_request_json(log_data=True): """ Used by endpoints which are expecting a JSON payload in the request body. Returns the JSON payload. From b988826ec91a3d633526e843e300c9251ba6cdbd Mon Sep 17 00:00:00 2001 From: Mitch Harding Date: Mon, 16 Dec 2024 16:38:56 -0500 Subject: [PATCH 2/4] Update requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f01eff6d..f46304f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ python-dateutil PyYAML redis requests -requests-retry-session>=0.2.2 +requests-retry-session>=2.0 urllib3 # The purpose of this file is to contain python runtime requirements From 871c8b27626053dc0570d34cf117a08cd2035b47 Mon Sep 17 00:00:00 2001 From: Mitch Harding Date: Mon, 16 Dec 2024 16:47:25 -0500 Subject: [PATCH 3/4] Update api/openapi.yaml.in --- api/openapi.yaml.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 5a2612b4..d7dae2f4 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1704,7 +1704,8 @@ paths: maximum: 1048576 in: query description: |- - Maximum number of Components to include in response. Used for paging. + Maximum number of Components to include in response. Used for paging. 0 means no limit + (which is the same as not specifying this parameter). description: |- Retrieve the full collection of Components in the form of a ComponentArray. Full results can also be filtered by query From 6dc56648e752e30b214676d91bcbb8e4029bc22f Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 16 Dec 2024 16:53:35 -0500 Subject: [PATCH 4/4] Update Python version in setup.py --- src/setup.py.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/setup.py.in b/src/setup.py.in index 4ccc325a..10b87e1e 100644 --- a/src/setup.py.in +++ b/src/setup.py.in @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -33,7 +33,7 @@ setup( packages=find_namespace_packages(), keywords="cray kubernetes boot orchestration", classifiers=[ - "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: Other/Proprietary License", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Systems Administration",