From bccce47cead903144dcf3b338d86a66a8260f918 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Tue, 17 Dec 2024 15:16:37 -0500 Subject: [PATCH] CASMCMS-9225: Move BOS client to new paradigm --- src/bos/common/clients/bos/__init__.py | 25 +++ src/bos/common/clients/bos/base.py | 121 ++++++++++ .../clients/bos/client.py} | 23 +- .../clients/bos/components.py | 18 +- .../utils => common}/clients/bos/options.py | 22 +- .../clients/bos/session_templates.py | 2 +- .../utils => common}/clients/bos/sessions.py | 7 + src/bos/common/clients/cfs/client.py | 2 +- src/bos/common/utils.py | 36 ++- src/bos/operators/actual_state_cleanup.py | 6 +- src/bos/operators/base.py | 25 ++- src/bos/operators/configuration.py | 4 +- src/bos/operators/discovery.py | 4 +- src/bos/operators/filters/filters.py | 6 +- src/bos/operators/power_off_forceful.py | 6 +- src/bos/operators/power_off_graceful.py | 4 +- src/bos/operators/power_on.py | 8 +- src/bos/operators/session_cleanup.py | 5 +- src/bos/operators/session_completion.py | 10 +- src/bos/operators/session_setup.py | 13 +- src/bos/operators/status.py | 7 +- src/bos/operators/utils/clients/bos/base.py | 210 ------------------ .../utils/clients/bos/sessions_status.py | 63 ------ 23 files changed, 285 insertions(+), 342 deletions(-) create mode 100644 src/bos/common/clients/bos/__init__.py create mode 100644 src/bos/common/clients/bos/base.py rename src/bos/{operators/utils/clients/bos/__init__.py => common/clients/bos/client.py} (70%) rename src/bos/{operators/utils => common}/clients/bos/components.py (71%) rename src/bos/{operators/utils => common}/clients/bos/options.py (76%) rename src/bos/{operators/utils => common}/clients/bos/session_templates.py (96%) rename src/bos/{operators/utils => common}/clients/bos/sessions.py (86%) delete mode 100644 src/bos/operators/utils/clients/bos/base.py delete mode 100644 src/bos/operators/utils/clients/bos/sessions_status.py diff --git a/src/bos/common/clients/bos/__init__.py b/src/bos/common/clients/bos/__init__.py new file mode 100644 index 00000000..2e6580aa --- /dev/null +++ b/src/bos/common/clients/bos/__init__.py @@ -0,0 +1,25 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from .client import BOSClient diff --git a/src/bos/common/clients/bos/base.py b/src/bos/common/clients/bos/base.py new file mode 100644 index 00000000..a73bec0d --- /dev/null +++ b/src/bos/common/clients/bos/base.py @@ -0,0 +1,121 @@ +# +# MIT License +# +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from abc import ABC +import logging + +from bos.common.clients.endpoints import BaseEndpoint +from bos.common.tenant_utils import get_new_tenant_header +from bos.common.utils import PROTOCOL + +LOGGER = logging.getLogger(__name__) + +API_VERSION = 'v2' +SERVICE_NAME = 'cray-bos' +BASE_BOS_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{API_VERSION}" + + +class BaseBosEndpoint(BaseEndpoint, ABC): + """ + This base class provides generic access to the BOS API. + The individual endpoint needs to be overridden for a specific endpoint. + """ + BASE_ENDPOINT = BASE_BOS_ENDPOINT + + +class BaseBosNonTenantAwareEndpoint(BaseBosEndpoint, ABC): + """ + This base class provides generic access to the BOS API for non-tenant-aware endpoints + The individual endpoint needs to be overridden for a specific endpoint. + """ + + def get_item(self, item_id): + """Get information for a single BOS item""" + return self.get(uri=item_id) + + def get_items(self, **kwargs): + """Get information for all BOS items""" + return self.get(params=kwargs) + + def update_item(self, item_id, data): + """Update information for a single BOS item""" + return self.patch(uri=item_id, json=data) + + def update_items(self, data): + """Update information for multiple BOS items""" + return self.patch(json=data) + + def put_items(self, data): + """Put information for multiple BOS Items""" + return self.put(json=data) + + def delete_items(self, **kwargs): + """Delete information for multiple BOS items""" + return self.delete(params=kwargs) + + +class BaseBosTenantAwareEndpoint(BaseBosEndpoint, ABC): + """ + This base class provides generic access to the BOS API for tenant aware endpoints. + The individual endpoint needs to be overridden for a specific endpoint. + """ + + def get_item(self, item_id, tenant): + """Get information for a single BOS item""" + return self.get(uri=item_id, headers=get_new_tenant_header(tenant)) + + def get_items(self, **kwargs): + """Get information for all BOS items""" + headers = None + if "tenant" in kwargs: + tenant = kwargs.pop("tenant") + headers = get_new_tenant_header(tenant) + return self.get(params=kwargs, headers=headers) + + def update_item(self, item_id, tenant, data): + """Update information for a single BOS item""" + return self.patch(uri=item_id, + json=data, + headers=get_new_tenant_header(tenant)) + + def update_items(self, tenant, data): + """Update information for multiple BOS items""" + return self.patch(json=data, headers=get_new_tenant_header(tenant)) + + def post_item(self, item_id, tenant, data=None): + """Post information for a single BOS item""" + return self.post(uri=item_id, + json=data, + headers=get_new_tenant_header(tenant)) + + def put_items(self, tenant, data): + """Put information for multiple BOS items""" + return self.put(json=data, headers=get_new_tenant_header(tenant)) + + def delete_items(self, **kwargs): + """Delete information for multiple BOS items""" + headers = None + if "tenant" in kwargs: + tenant = kwargs.pop("tenant") + headers = get_new_tenant_header(tenant) + return self.delete(params=kwargs, headers=headers) diff --git a/src/bos/operators/utils/clients/bos/__init__.py b/src/bos/common/clients/bos/client.py similarity index 70% rename from src/bos/operators/utils/clients/bos/__init__.py rename to src/bos/common/clients/bos/client.py index 04c4e94f..8c411b14 100644 --- a/src/bos/operators/utils/clients/bos/__init__.py +++ b/src/bos/common/clients/bos/client.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -21,16 +21,23 @@ # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. # +from bos.common.clients.api_client import APIClient + from .components import ComponentEndpoint from .sessions import SessionEndpoint from .session_templates import SessionTemplateEndpoint -from .sessions_status import SessionStatusEndpoint -class BOSClient: +class BOSClient(APIClient): + + @property + def components(self) -> ComponentEndpoint: + return self.get_endpoint(ComponentEndpoint) + + @property + def sessions(self) -> SessionEndpoint: + return self.get_endpoint(SessionEndpoint) - def __init__(self): - self.components = ComponentEndpoint() - self.sessions = SessionEndpoint() - self.session_status = SessionStatusEndpoint() - self.session_templates = SessionTemplateEndpoint() + @property + def session_templates(self) -> SessionTemplateEndpoint: + return self.get_endpoint(SessionTemplateEndpoint) diff --git a/src/bos/operators/utils/clients/bos/components.py b/src/bos/common/clients/bos/components.py similarity index 71% rename from src/bos/operators/utils/clients/bos/components.py rename to src/bos/common/clients/bos/components.py index 049b9269..e8681839 100644 --- a/src/bos/operators/utils/clients/bos/components.py +++ b/src/bos/common/clients/bos/components.py @@ -23,19 +23,31 @@ # import logging -from .base import BaseBosEndpoint +from .base import BaseBosNonTenantAwareEndpoint +from .options import options LOGGER = logging.getLogger(__name__) -class ComponentEndpoint(BaseBosEndpoint): +class ComponentEndpoint(BaseBosNonTenantAwareEndpoint): ENDPOINT = __name__.lower().rsplit('.', maxsplit=1)[-1] def get_component(self, component_id): return self.get_item(component_id) def get_components(self, **kwargs): - return self.get_items(**kwargs) + page_size = options.max_component_batch_size + if page_size == 0: + return self.get_items(**kwargs) + next_page = self.get_items(page_size=page_size, **kwargs) + results = next_page + while len(next_page) == page_size: + last_id = next_page[-1]["id"] + next_page = self.get_items(page_size=page_size, + start_after_id=last_id, + **kwargs) + results.extend(next_page) + return results def update_component(self, component_id, data): return self.update_item(component_id, data) diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/common/clients/bos/options.py similarity index 76% rename from src/bos/operators/utils/clients/bos/options.py rename to src/bos/common/clients/bos/options.py index 9b604a3f..168e2582 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/common/clients/bos/options.py @@ -23,17 +23,20 @@ # import logging import json +from typing import Optional -from requests.exceptions import HTTPError, ConnectionError +import requests +from requests.exceptions import HTTPError +from requests.exceptions import ConnectionError as RequestsConnectionError from urllib3.exceptions import MaxRetryError from bos.common.options import OptionsCache -from bos.common.utils import exc_type_msg, requests_retry_session -from bos.operators.utils.clients.bos.base import BASE_ENDPOINT +from bos.common.utils import exc_type_msg, retry_session_get +from bos.common.clients.bos.base import BASE_BOS_ENDPOINT LOGGER = logging.getLogger(__name__) __name = __name__.lower().rsplit('.', maxsplit=1)[-1] -ENDPOINT = f"{BASE_ENDPOINT}/{__name}" +ENDPOINT = f"{BASE_BOS_ENDPOINT}/{__name}" class Options(OptionsCache): @@ -44,15 +47,14 @@ class Options(OptionsCache): result in network calls. """ - def _get_options(self) -> dict: + def _get_options(self, session: Optional[requests.Session] = None) -> dict: """Retrieves the current options from the BOS api""" - session = requests_retry_session() LOGGER.debug("GET %s", ENDPOINT) try: - response = session.get(ENDPOINT) - response.raise_for_status() - return json.loads(response.text) - except (ConnectionError, MaxRetryError) as e: + with retry_session_get(ENDPOINT, session=session) as response: + response.raise_for_status() + return json.loads(response.text) + except (RequestsConnectionError, MaxRetryError) as e: LOGGER.error("Unable to connect to BOS: %s", exc_type_msg(e)) except HTTPError as e: LOGGER.error("Unexpected response from BOS: %s", exc_type_msg(e)) diff --git a/src/bos/operators/utils/clients/bos/session_templates.py b/src/bos/common/clients/bos/session_templates.py similarity index 96% rename from src/bos/operators/utils/clients/bos/session_templates.py rename to src/bos/common/clients/bos/session_templates.py index 5d9fae11..09c5cec4 100644 --- a/src/bos/operators/utils/clients/bos/session_templates.py +++ b/src/bos/common/clients/bos/session_templates.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), diff --git a/src/bos/operators/utils/clients/bos/sessions.py b/src/bos/common/clients/bos/sessions.py similarity index 86% rename from src/bos/operators/utils/clients/bos/sessions.py rename to src/bos/common/clients/bos/sessions.py index fdbc4b97..6d5b7c17 100644 --- a/src/bos/operators/utils/clients/bos/sessions.py +++ b/src/bos/common/clients/bos/sessions.py @@ -42,3 +42,10 @@ def update_session(self, session_id, tenant, data): def delete_sessions(self, **kwargs): return self.delete_items(**kwargs) + + def post_session_status(self, session_id, tenant): + """ + Post information for a single BOS Session status. + This basically saves the BOS Session status to the database. + """ + return self.post_item(f'{session_id}/status', tenant) diff --git a/src/bos/common/clients/cfs/client.py b/src/bos/common/clients/cfs/client.py index 2c2f4508..68f05bb0 100644 --- a/src/bos/common/clients/cfs/client.py +++ b/src/bos/common/clients/cfs/client.py @@ -22,7 +22,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # from bos.common.clients.api_client import APIClient -from bos.operators.utils.clients.bos.options import options +from bos.common.clients.bos.options import options from .components import ComponentEndpoint diff --git a/src/bos/common/utils.py b/src/bos/common/utils.py index 69dd84c0..c2aecf68 100644 --- a/src/bos/common/utils.py +++ b/src/bos/common/utils.py @@ -23,14 +23,16 @@ # # Standard imports +from contextlib import nullcontext import datetime from functools import partial import re import traceback -from typing import List, Unpack +from typing import Iterator, List, Optional, Unpack # Third party imports from dateutil.parser import parse +import requests import requests_retry_session as rrs PROTOCOL = 'http' @@ -76,6 +78,11 @@ def duration_to_timedelta(timestamp: str): read_timeout=10) +retry_session_manager = partial(rrs.retry_session_manager, + protocol=PROTOCOL, + **DEFAULT_RETRY_ADAPTER_ARGS) + + class RetrySessionManager(rrs.RetrySessionManager): """ Just sets the default values we use for our requests sessions @@ -90,7 +97,32 @@ def __init__(self, super().__init__(protocol=protocol, **adapter_kwargs) -requests_retry_session = partial(rrs.requests_retry_session, +def retry_session( + session: Optional[requests.Session] = None, + protocol: Optional[str] = None, + adapter_kwargs: Optional[rrs.RequestsRetryAdapterArgs] = None +) -> Iterator[requests.Session]: + if session is not None: + return nullcontext(session) + kwargs = adapter_kwargs or {} + if protocol is not None: + return retry_session_manager(protocol=protocol, **kwargs) # pylint: disable=redundant-keyword-arg + return retry_session_manager(**kwargs) + + +def retry_session_get(*get_args, + session: Optional[requests.Session] = None, + protocol: Optional[str] = None, + adapter_kwargs: Optional[ + rrs.RequestsRetryAdapterArgs] = None, + **get_kwargs) -> Iterator[requests.Response]: + with retry_session(session=session, + protocol=protocol, + adapter_kwargs=adapter_kwargs) as _session: + return _session.get(*get_args, **get_kwargs) + + +requests_retry_session = partial(rrs.requests_retry_session, session=None, protocol=PROTOCOL, **DEFAULT_RETRY_ADAPTER_ARGS) diff --git a/src/bos/operators/actual_state_cleanup.py b/src/bos/operators/actual_state_cleanup.py index f6f5cf9b..22e41a83 100644 --- a/src/bos/operators/actual_state_cleanup.py +++ b/src/bos/operators/actual_state_cleanup.py @@ -24,9 +24,9 @@ # import logging +from bos.common.clients.bos.options import options from bos.common.utils import duration_to_timedelta from bos.common.values import EMPTY_ACTUAL_STATE -from bos.operators.utils.clients.bos.options import options from bos.operators.base import BaseOperator, main from bos.operators.filters import BOSQuery, ActualStateAge, ActualBootStateIsSet @@ -53,7 +53,7 @@ def name(self): @property def filters(self): return [ - BOSQuery(), + self.BOSQuery(), ActualBootStateIsSet(), ActualStateAge(seconds=duration_to_timedelta( options.component_actual_state_ttl).total_seconds()) @@ -69,7 +69,7 @@ def _act(self, components): if data: LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Calling to update with payload: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) return components diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index 0bfc8ebc..fa09c45b 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -35,15 +35,15 @@ import time from typing import Generator, List, NoReturn, Type +from bos.common.clients.bos import BOSClient +from bos.common.clients.bos.options import options from bos.common.clients.bss import BSSClient from bos.common.clients.cfs import CFSClient from bos.common.clients.pcs import PCSClient from bos.common.utils import exc_type_msg from bos.common.values import Status -from bos.operators.filters import DesiredConfigurationSetInCFS +from bos.operators.filters import BOSQuery, DesiredConfigurationSetInCFS from bos.operators.filters.base import BaseFilter -from bos.operators.utils.clients.bos.options import options -from bos.operators.utils.clients.bos import BOSClient from bos.operators.utils.liveness.timestamp import Timestamp LOGGER = logging.getLogger(__name__) @@ -67,7 +67,7 @@ class ApiClients: """ def __init__(self): - #self.bos = BOSClient() + self.bos = BOSClient() self.bss = BSSClient() self.cfs = CFSClient() #self.hsm = HSMClient() @@ -79,7 +79,7 @@ def __enter__(self): """ Enter context for all API clients """ - #self._stack.enter_context(self.bos) + self._stack.enter_context(self.bos) self._stack.enter_context(self.bss) self._stack.enter_context(self.cfs) #self._stack.enter_context(self.hsm) @@ -114,7 +114,6 @@ class BaseOperator(ABC): frequency_option = "polling_frequency" def __init__(self) -> NoReturn: - self.bos_client = BOSClient() self.__max_batch_size = 0 self._client: ApiClients | None = None @@ -138,6 +137,14 @@ def name(self) -> str: def filters(self) -> List[Type[BaseFilter]]: return [] + def BOSQuery(self, **kwargs) -> BOSQuery: + """ + Shortcut to get a BOSQuery filter with the bos_client for this operator + """ + if 'bos_client' not in kwargs: + kwargs['bos_client'] = self.client.bos + return BOSQuery(**kwargs) + @property def DesiredConfigurationSetInCFS(self) -> DesiredConfigurationSetInCFS: """ @@ -308,7 +315,7 @@ def _update_database(self, data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) def _preset_last_action(self, components: List[dict]) -> None: # This is done to eliminate the window between performing an action and marking the @@ -331,7 +338,7 @@ def _preset_last_action(self, components: List[dict]) -> None: data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) def _update_database_for_failure(self, components: List[dict]) -> None: """ @@ -358,7 +365,7 @@ def _update_database_for_failure(self, components: List[dict]) -> None: data.append(patch) LOGGER.info('Found %d components that require updates', len(data)) LOGGER.debug('Updated components: %s', data) - self.bos_client.components.update_components(data) + self.client.bos.components.update_components(data) def chunk_components(components: List[dict], diff --git a/src/bos/operators/configuration.py b/src/bos/operators/configuration.py index 816ce240..5f9d142a 100644 --- a/src/bos/operators/configuration.py +++ b/src/bos/operators/configuration.py @@ -26,7 +26,7 @@ from bos.common.values import Status from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, NOT +from bos.operators.filters import NOT LOGGER = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=Status.configuring), + self.BOSQuery(enabled=True, status=Status.configuring), NOT(self.DesiredConfigurationSetInCFS) ] diff --git a/src/bos/operators/discovery.py b/src/bos/operators/discovery.py index eecad813..ff010474 100644 --- a/src/bos/operators/discovery.py +++ b/src/bos/operators/discovery.py @@ -84,7 +84,7 @@ def _run(self) -> None: return LOGGER.info("%s new component(s) from HSM.", len(components_to_add)) for chunk in self._chunk_components(components_to_add): - self.bos_client.components.put_components(chunk) + self.client.bos.components.put_components(chunk) LOGGER.info("%s new component(s) added to BOS!", len(chunk)) @property @@ -93,7 +93,7 @@ def bos_components(self) -> Set[str]: The set of components currently known to BOS """ components = set() - for component in self.bos_client.components.get_components(): + for component in self.client.bos.components.get_components(): components.add(component['id']) return components diff --git a/src/bos/operators/filters/filters.py b/src/bos/operators/filters/filters.py index 882971f2..45f6c944 100644 --- a/src/bos/operators/filters/filters.py +++ b/src/bos/operators/filters/filters.py @@ -28,10 +28,10 @@ import re from typing import List, Type +from bos.common.clients.bos import BOSClient from bos.common.clients.cfs import CFSClient from bos.common.utils import get_current_time, load_timestamp from bos.operators.filters.base import BaseFilter, DetailsFilter, IDFilter, LocalFilter -from bos.operators.utils.clients.bos import BOSClient from bos.operators.utils.clients.hsm import get_components as get_hsm_components LOGGER = logging.getLogger(__name__) @@ -68,14 +68,14 @@ class BOSQuery(DetailsFilter): """Gets all components from BOS that match the kwargs """ INITIAL: bool = True - def __init__(self, **kwargs) -> None: + def __init__(self, bos_client: BOSClient, **kwargs) -> None: """ Init for the BOSQuery filter kwargs corresponds to arguments for the BOS get_components method """ super().__init__() self.kwargs = kwargs - self.bos_client = BOSClient() + self.bos_client = bos_client def _filter(self, _) -> List[dict]: return self.bos_client.components.get_components(**self.kwargs) diff --git a/src/bos/operators/power_off_forceful.py b/src/bos/operators/power_off_forceful.py index 365712c3..5dd69641 100644 --- a/src/bos/operators/power_off_forceful.py +++ b/src/bos/operators/power_off_forceful.py @@ -24,10 +24,10 @@ # import logging +from bos.common.clients.bos.options import options from bos.common.values import Action, Status -from bos.operators.utils.clients.bos.options import options from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, HSMState, TimeSinceLastAction +from bos.operators.filters import HSMState, TimeSinceLastAction LOGGER = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, + self.BOSQuery(enabled=True, status=','.join([ Status.power_off_forcefully_called, Status.power_off_gracefully_called diff --git a/src/bos/operators/power_off_graceful.py b/src/bos/operators/power_off_graceful.py index f9e6b8a1..c75bc497 100644 --- a/src/bos/operators/power_off_graceful.py +++ b/src/bos/operators/power_off_graceful.py @@ -26,7 +26,7 @@ from bos.common.values import Action, Status from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, HSMState +from bos.operators.filters import HSMState LOGGER = logging.getLogger(__name__) @@ -47,7 +47,7 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=Status.power_off_pending), + self.BOSQuery(enabled=True, status=Status.power_off_pending), HSMState(), ] diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index 2f2e57bc..99eb4387 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -35,9 +35,9 @@ from bos.common.utils import exc_type_msg, get_image_id_from_kernel, \ using_sbps_check_kernel_parameters, components_by_id from bos.common.values import Action, Status -from bos.operators.utils.clients.ims import tag_image from bos.operators.base import BaseOperator, main -from bos.operators.filters import BOSQuery, HSMState +from bos.operators.filters import HSMState +from bos.operators.utils.clients.ims import tag_image from bos.server.dbs.boot_artifacts import record_boot_artifacts LOGGER = logging.getLogger(__name__) @@ -60,7 +60,7 @@ def name(self): @property def filters(self): return [ - BOSQuery(enabled=True, status=Status.power_on_pending), + self.BOSQuery(enabled=True, status=Status.power_on_pending), HSMState() ] @@ -194,7 +194,7 @@ def _set_bss(self, boot_artifacts, bos_sessions, retries=5): } for comp in bss_tokens] LOGGER.debug('Updated components (minus desired_state data): %s', redacted_component_updates) - self.bos_client.components.update_components(bss_tokens) + self.client.bos.components.update_components(bss_tokens) def _tag_images(self, boot_artifacts: Dict[Tuple[str, str, str], Set[str]], components: List[dict]) -> None: diff --git a/src/bos/operators/session_cleanup.py b/src/bos/operators/session_cleanup.py index d7bdae2e..34e3366c 100644 --- a/src/bos/operators/session_cleanup.py +++ b/src/bos/operators/session_cleanup.py @@ -25,8 +25,9 @@ import logging import re +from bos.common.clients.bos.options import options from bos.operators.base import BaseOperator, main -from bos.operators.utils.clients.bos.options import options + LOGGER = logging.getLogger(__name__) @@ -68,7 +69,7 @@ def _run(self) -> None: if self.disabled: return - self.bos_client.sessions.delete_sessions( + self.client.bos.sessions.delete_sessions( **{ 'status': 'complete', 'min_age': options.cleanup_completed_session_ttl diff --git a/src/bos/operators/session_completion.py b/src/bos/operators/session_completion.py index 23af049f..8bc61c58 100644 --- a/src/bos/operators/session_completion.py +++ b/src/bos/operators/session_completion.py @@ -59,24 +59,24 @@ def _run(self) -> None: session.get("tenant")) def _get_incomplete_sessions(self): - return self.bos_client.sessions.get_sessions(status='running') + return self.client.bos.sessions.get_sessions(status='running') def _get_incomplete_components(self, session_id): - components = self.bos_client.components.get_components( + components = self.client.bos.components.get_components( session=session_id, enabled=True) - components += self.bos_client.components.get_components( + components += self.client.bos.components.get_components( staged_session=session_id) return components def _mark_session_complete(self, session_id, tenant): - self.bos_client.sessions.update_session(session_id, tenant, { + self.client.bos.sessions.update_session(session_id, tenant, { 'status': { 'status': 'complete', 'end_time': get_current_timestamp() } }) # This call causes the session status to saved in the database. - self.bos_client.session_status.post_session_status(session_id, tenant) + self.client.bos.sessions.post_session_status(session_id, tenant) LOGGER.info('Session %s is complete', session_id) diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 270f7783..c24eb9ae 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -27,17 +27,18 @@ from typing import Set from botocore.exceptions import ClientError +from bos.common.clients.bos.options import options +from bos.common.tenant_utils import get_tenant_component_set, InvalidTenantException +from bos.common.utils import exc_type_msg +from bos.common.values import Action, EMPTY_ACTUAL_STATE, EMPTY_DESIRED_STATE, EMPTY_STAGED_STATE from bos.operators.base import BaseOperator, main, chunk_components from bos.operators.filters.filters import HSMState from bos.operators.utils.clients.hsm import Inventory from bos.operators.utils.clients.s3 import S3Object, S3ObjectNotFound from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory -from bos.operators.utils.clients.bos.options import options from bos.operators.utils.rootfs.factory import ProviderFactory from bos.operators.session_completion import SessionCompletionOperator -from bos.common.utils import exc_type_msg -from bos.common.values import Action, EMPTY_ACTUAL_STATE, EMPTY_DESIRED_STATE, EMPTY_STAGED_STATE -from bos.common.tenant_utils import get_tenant_component_set, InvalidTenantException + LOGGER = logging.getLogger(__name__) @@ -73,11 +74,11 @@ def _run(self) -> None: LOGGER.info('Found %d sessions that require action', len(sessions)) inventory_cache = Inventory() for data in sessions: - session = Session(data, inventory_cache, self.bos_client) + session = Session(data, inventory_cache, self.client.bos) session.setup(self.max_batch_size) def _get_pending_sessions(self): - return self.bos_client.sessions.get_sessions(status='pending') + return self.client.bos.sessions.get_sessions(status='pending') class Session: diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index 874ede67..e701398e 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -24,11 +24,12 @@ # import logging +from bos.common.clients.bos.options import options from bos.common.values import Phase, Status, Action, EMPTY_ACTUAL_STATE from bos.operators.base import BaseOperator, main from bos.operators.filters import DesiredBootStateIsOff, BootArtifactStatesMatch, \ DesiredConfigurationIsNone, LastActionIs, TimeSinceLastAction -from bos.operators.utils.clients.bos.options import options + LOGGER = logging.getLogger(__name__) @@ -74,7 +75,7 @@ def _act(self, components): def _run(self) -> None: """ A single pass of detecting and acting on components """ - components = self.bos_client.components.get_components(enabled=True) + components = self.client.bos.components.get_components(enabled=True) if not components: LOGGER.debug('No enabled components found') return @@ -110,7 +111,7 @@ def _run_on_chunk(self, components) -> None: LOGGER.info('Found %d components that require status updates', len(updated_components)) LOGGER.debug('Updated components: %s', updated_components) - self.bos_client.components.update_components(updated_components) + self.client.bos.components.update_components(updated_components) def _get_cfs_components(self): """ diff --git a/src/bos/operators/utils/clients/bos/base.py b/src/bos/operators/utils/clients/bos/base.py deleted file mode 100644 index bbdb0fab..00000000 --- a/src/bos/operators/utils/clients/bos/base.py +++ /dev/null @@ -1,210 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import json -import logging -from requests.exceptions import HTTPError, ConnectionError -from urllib3.exceptions import MaxRetryError - -from bos.common.tenant_utils import get_new_tenant_header -from bos.common.utils import PROTOCOL, exc_type_msg, requests_retry_session - -LOGGER = logging.getLogger(__name__) - -API_VERSION = 'v2' -SERVICE_NAME = 'cray-bos' -BASE_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{API_VERSION}" - - -def log_call_errors(func): - - def wrap(*args, **kwargs): - try: - result = func(*args, **kwargs) - return result - except (ConnectionError, MaxRetryError) as e: - LOGGER.error("Unable to connect to BOS: %s", exc_type_msg(e)) - raise e - except HTTPError as e: - LOGGER.error("Unexpected response from BOS: %s", exc_type_msg(e)) - raise e - except json.JSONDecodeError as e: - LOGGER.error("Non-JSON response from BOS: %s", exc_type_msg(e)) - raise e - - return wrap - - -class BaseBosEndpoint: - """ - This base class provides generic access to the BOS API. - The individual endpoint needs to be overridden for a specific endpoint. - """ - ENDPOINT = '' - - def __init__(self): - self.base_url = f"{BASE_ENDPOINT}/{self.ENDPOINT}" - self.session = requests_retry_session() - - @log_call_errors - def get_item(self, item_id): - """Get information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("GET %s", url) - response = self.session.get(url) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def get_items(self, **kwargs): - """Get information for all BOS items""" - LOGGER.debug("GET %s with params=%s", self.base_url, kwargs) - response = self.session.get(self.base_url, params=kwargs) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def update_item(self, item_id, data): - """Update information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("PATCH %s with body=%s", url, data) - response = self.session.patch(url, json=data) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def update_items(self, data): - """Update information for multiple BOS items""" - LOGGER.debug("PATCH %s with body=%s", self.base_url, data) - response = self.session.patch(self.base_url, json=data) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def put_items(self, data): - """Put information for multiple BOS Items""" - LOGGER.debug("PUT %s with body=%s", self.base_url, data) - response = self.session.put(self.base_url, json=data) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def delete_items(self, **kwargs): - """Delete information for multiple BOS items""" - LOGGER.debug("DELETE %s with params=%s", self.base_url, kwargs) - response = self.session.delete(self.base_url, params=kwargs) - response.raise_for_status() - return json.loads(response.text) if response.text else None - - -class BaseBosTenantAwareEndpoint(BaseBosEndpoint): - """ - This base class provides generic access to the BOS API for tenant aware endpoints. - The individual endpoint needs to be overridden for a specific endpoint. - """ - - @log_call_errors - def get_item(self, item_id, tenant): - """Get information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("GET %s for tenant=%s", url, tenant) - response = self.session.get(url, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def get_items(self, **kwargs): - """Get information for all BOS items""" - headers = None - if "tenant" in kwargs: - tenant = kwargs.pop("tenant") - headers = get_new_tenant_header(tenant) - LOGGER.debug("GET %s for tenant=%s with params=%s", self.base_url, - tenant, kwargs) - else: - LOGGER.debug("GET %s with params=%s", self.base_url, kwargs) - response = self.session.get(self.base_url, - params=kwargs, - headers=headers) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def update_item(self, item_id, tenant, data): - """Update information for a single BOS item""" - url = self.base_url + '/' + item_id - LOGGER.debug("PATCH %s for tenant=%s with body=%s", url, tenant, data) - response = self.session.patch(url, - json=data, - headers=get_new_tenant_header(tenant)) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def update_items(self, tenant, data): - """Update information for multiple BOS items""" - LOGGER.debug("PATCH %s for tenant=%s with body=%s", self.base_url, - tenant, data) - response = self.session.patch(self.base_url, - json=data, - headers=get_new_tenant_header(tenant)) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def put_items(self, tenant, data): - """Put information for multiple BOS items""" - LOGGER.debug("PUT %s for tenant=%s with body=%s", self.base_url, - tenant, data) - response = self.session.put(self.base_url, - json=data, - headers=get_new_tenant_header(tenant)) - response.raise_for_status() - items = json.loads(response.text) - return items - - @log_call_errors - def delete_items(self, **kwargs): - """Delete information for multiple BOS items""" - headers = None - if "tenant" in kwargs: - tenant = kwargs.pop("tenant") - headers = get_new_tenant_header(tenant) - LOGGER.debug("DELETE %s for tenant=%s with params=%s", - self.base_url, tenant, kwargs) - else: - LOGGER.debug("DELETE %s with params=%s", self.base_url, kwargs) - response = self.session.delete(self.base_url, - params=kwargs, - headers=headers) - response.raise_for_status() - return json.loads(response.text) if response.text else None diff --git a/src/bos/operators/utils/clients/bos/sessions_status.py b/src/bos/operators/utils/clients/bos/sessions_status.py deleted file mode 100644 index b55c28bc..00000000 --- a/src/bos/operators/utils/clients/bos/sessions_status.py +++ /dev/null @@ -1,63 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# -import json -import logging - -from bos.common.tenant_utils import get_new_tenant_header -from bos.common.utils import requests_retry_session -from .base import BASE_ENDPOINT, log_call_errors - -LOGGER = logging.getLogger(__name__) - - -class SessionStatusEndpoint: - ENDPOINT = 'sessions' - - def __init__(self): - self.base_url = f"{BASE_ENDPOINT}/{self.ENDPOINT}" - - @log_call_errors - def get_session_status(self, session_id, tenant): - """Get information for a single BOS item""" - url = self.base_url + '/' + session_id + '/status' - session = requests_retry_session() - LOGGER.debug("GET %s for tenant=%s", url, tenant) - response = session.get(url, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - item = json.loads(response.text) - return item - - @log_call_errors - def post_session_status(self, session_id, tenant): - """ - Post information for a single BOS Session status. - This basically saves the BOS Session status to the database. - """ - session = requests_retry_session() - url = self.base_url + '/' + session_id + '/status' - LOGGER.debug("POST %s for tenant=%s", url, tenant) - response = session.post(url, headers=get_new_tenant_header(tenant)) - response.raise_for_status() - items = json.loads(response.text) - return items