diff --git a/CHANGELOG.md b/CHANGELOG.md index dec0dcfc..b597b885 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added basic paging ability for `GET` requests for `components`. +### Changed +- Modified operators to use paging when requesting BOS components, using a page size equal to the `max_components_batch_size` option. +- Put all requests code into context managers -- this includes the HTTP adapters, the sessions, and the request responses. + ## [2.31.0] - 2024-11-01 ### Removed - Moved BOS reporter to https://github.com/Cray-HPE/bos-reporter diff --git a/src/bos/common/clients/endpoints/request_error_handler.py b/src/bos/common/clients/endpoints/request_error_handler.py index 4de38993..c0c12803 100644 --- a/src/bos/common/clients/endpoints/request_error_handler.py +++ b/src/bos/common/clients/endpoints/request_error_handler.py @@ -42,6 +42,7 @@ class BaseRequestErrorHandler(ABC): """ The abstract base class for request error handlers that will be used by an API endpoint. """ + @classmethod @abstractmethod def handle_exception(cls, err: Exception, @@ -53,6 +54,7 @@ class RequestErrorHandler(BaseRequestErrorHandler): """ The default request error handler used by API endpoints. """ + @classmethod def handle_api_response_error(cls, err: ApiResponseError, request_data: RequestData) -> NoReturn: diff --git a/src/bos/common/tenant_utils.py b/src/bos/common/tenant_utils.py index 9a8aefd2..d41f16fc 100644 --- a/src/bos/common/tenant_utils.py +++ b/src/bos/common/tenant_utils.py @@ -25,10 +25,12 @@ import functools import logging import hashlib +from typing import Optional import connexion +import requests from requests.exceptions import HTTPError -from bos.common.utils import exc_type_msg, requests_retry_session, PROTOCOL +from bos.common.utils import exc_type_msg, retry_session_get, PROTOCOL LOGGER = logging.getLogger(__name__) @@ -73,21 +75,19 @@ def get_tenant_aware_key(key, tenant): return f"{tenant_hash}-{key_hash}" -def get_tenant_data(tenant, session=None): - if not session: - session = requests_retry_session() +def get_tenant_data(tenant, session: Optional[requests.Session] = None): url = f"{TENANT_ENDPOINT}/{tenant}" - response = session.get(url) - try: - response.raise_for_status() - except HTTPError as e: - LOGGER.error("Failed getting tenant data from tapms: %s", - exc_type_msg(e)) - if response.status_code == 404: - raise InvalidTenantException( - f"Data not found for tenant {tenant}") from e - raise - return response.json() + with retry_session_get(url, session=session) as response: + try: + response.raise_for_status() + except HTTPError as e: + LOGGER.error("Failed getting tenant data from tapms: %s", + exc_type_msg(e)) + if response.status_code == 404: + raise InvalidTenantException( + f"Data not found for tenant {tenant}") from e + raise + return response.json() def get_tenant_component_set(tenant: str) -> set: diff --git a/src/bos/common/utils.py b/src/bos/common/utils.py index c2aecf68..331d8f5f 100644 --- a/src/bos/common/utils.py +++ b/src/bos/common/utils.py @@ -28,7 +28,7 @@ from functools import partial import re import traceback -from typing import Iterator, List, Optional, Unpack +from typing import Iterator, Optional, Unpack # Third party imports from dateutil.parser import parse @@ -77,7 +77,6 @@ def duration_to_timedelta(timestamp: str): connect_timeout=3, read_timeout=10) - retry_session_manager = partial(rrs.retry_session_manager, protocol=PROTOCOL, **DEFAULT_RETRY_ADAPTER_ARGS) @@ -122,12 +121,6 @@ def retry_session_get(*get_args, return _session.get(*get_args, **get_kwargs) -requests_retry_session = partial(rrs.requests_retry_session, - session=None, - protocol=PROTOCOL, - **DEFAULT_RETRY_ADAPTER_ARGS) - - def compact_response_text(response_text: str) -> str: """ Often JSON is "pretty printed" in response text, which is undesirable for our logging. @@ -201,7 +194,7 @@ def using_sbps_check_kernel_parameters(kernel_parameters: str) -> bool: return "root=sbps-s3" in kernel_parameters -def components_by_id(components: List[dict]) -> dict: +def components_by_id(components: list[dict]) -> dict: """ Input: * components: a list containing individual components @@ -215,7 +208,7 @@ def components_by_id(components: List[dict]) -> dict: return {component["id"]: component for component in components} -def reverse_components_by_id(components_by_id_map: dict) -> List[dict]: +def reverse_components_by_id(components_by_id_map: dict) -> list[dict]: """ Input: components_by_id_map: a dictionary with the name of each component as the diff --git a/src/bos/operators/base.py b/src/bos/operators/base.py index d5b92927..025a7617 100644 --- a/src/bos/operators/base.py +++ b/src/bos/operators/base.py @@ -62,6 +62,7 @@ class MissingSessionData(BaseOperatorException): desired state. """ + class ApiClients: """ Context manager to provide API clients to BOS operators. @@ -97,7 +98,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): return self._stack.__exit__(exc_type, exc_val, exc_tb) - class BaseOperator(ABC): """ An abstract class for all BOS operators. diff --git a/src/bos/operators/power_off_forceful.py b/src/bos/operators/power_off_forceful.py index 911be7b9..110a7a81 100644 --- a/src/bos/operators/power_off_forceful.py +++ b/src/bos/operators/power_off_forceful.py @@ -50,10 +50,10 @@ def name(self): def filters(self): return [ self.BOSQuery(enabled=True, - status=','.join([ - Status.power_off_forcefully_called, - Status.power_off_gracefully_called - ])), + status=','.join([ + Status.power_off_forcefully_called, + Status.power_off_gracefully_called + ])), TimeSinceLastAction(seconds=options.max_power_off_wait_time), self.HSMState(), ] diff --git a/src/bos/operators/power_on.py b/src/bos/operators/power_on.py index d31b44da..62c98a4f 100644 --- a/src/bos/operators/power_on.py +++ b/src/bos/operators/power_on.py @@ -80,7 +80,9 @@ def _act(self, components: Union[List[dict], None]): raise Exception( f"Error encountered setting BSS information: {e}") from e try: - self.client.cfs.components.set_cfs(components, enabled=False, clear_state=True) + self.client.cfs.components.set_cfs(components, + enabled=False, + clear_state=True) except Exception as e: raise Exception( f"Error encountered setting CFS information: {e}") from e @@ -148,10 +150,11 @@ def _set_bss(self, boot_artifacts, bos_sessions, retries=5): for key, nodes in boot_artifacts.items(): kernel, kernel_parameters, initrd = key try: - resp = self.client.bss.boot_parameters.set_bss(node_set=nodes, - kernel_params=kernel_parameters, - kernel=kernel, - initrd=initrd) + resp = self.client.bss.boot_parameters.set_bss( + node_set=nodes, + kernel_params=kernel_parameters, + kernel=kernel, + initrd=initrd) resp.raise_for_status() except HTTPError as err: LOGGER.error( @@ -237,7 +240,8 @@ def _tag_images(self, boot_artifacts: Dict[Tuple[str, str, str], Set[str]], my_components_by_id = components_by_id(components) for image in image_ids: try: - self.client.ims.images.tag_image(image, "set", "sbps-project", "true") + self.client.ims.images.tag_image(image, "set", "sbps-project", + "true") except Exception as e: components_to_update = [] for node in image_id_to_nodes[image]: diff --git a/src/bos/operators/session_cleanup.py b/src/bos/operators/session_cleanup.py index 34e3366c..d6632bf4 100644 --- a/src/bos/operators/session_cleanup.py +++ b/src/bos/operators/session_cleanup.py @@ -28,7 +28,6 @@ from bos.common.clients.bos.options import options from bos.operators.base import BaseOperator, main - LOGGER = logging.getLogger(__name__) diff --git a/src/bos/operators/session_setup.py b/src/bos/operators/session_setup.py index 99daa75d..7323de33 100644 --- a/src/bos/operators/session_setup.py +++ b/src/bos/operators/session_setup.py @@ -40,7 +40,6 @@ from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory from bos.operators.utils.rootfs.factory import ProviderFactory - LOGGER = logging.getLogger(__name__) @@ -75,7 +74,8 @@ def _run(self) -> None: LOGGER.info('Found %d sessions that require action', len(sessions)) inventory_cache = Inventory(self.client.hsm) for data in sessions: - session = Session(data, inventory_cache, self.client.bos, self.HSMState) + session = Session(data, inventory_cache, self.client.bos, + self.HSMState) session.setup(self.max_batch_size) def _get_pending_sessions(self): @@ -84,7 +84,8 @@ def _get_pending_sessions(self): class Session: - def __init__(self, data, inventory_cache, bos_client: BOSClient, hsm_state: Callable[...,HSMState]): + def __init__(self, data, inventory_cache, bos_client: BOSClient, + hsm_state: Callable[..., HSMState]): self.session_data = data self.inventory = inventory_cache self.bos_client = bos_client diff --git a/src/bos/operators/status.py b/src/bos/operators/status.py index e701398e..7ee46760 100644 --- a/src/bos/operators/status.py +++ b/src/bos/operators/status.py @@ -30,7 +30,6 @@ from bos.operators.filters import DesiredBootStateIsOff, BootArtifactStatesMatch, \ DesiredConfigurationIsNone, LastActionIs, TimeSinceLastAction - LOGGER = logging.getLogger(__name__) diff --git a/src/bos/server/controllers/v2/components.py b/src/bos/server/controllers/v2/components.py index 0ab44bca..0677030c 100644 --- a/src/bos/server/controllers/v2/components.py +++ b/src/bos/server/controllers/v2/components.py @@ -101,14 +101,15 @@ def get_v2_components_data(id_list=None, Allows filtering using a comma separated list of ids. """ - tenant_components = None if tenant is None else get_tenant_component_set(tenant) + tenant_components = None if tenant is None else get_tenant_component_set( + tenant) if id_list is not None: id_set = set(id_list) if tenant_components is not None: id_set.intersection_update(tenant_components) else: - id_set = tenant_components + id_set = tenant_components # If id_set is not None but is empty, that means no components in the system # will match our filter, so we can return an empty list immediately.