CASMCMS-9225: CHANGELOG update, linting, update utils

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Added basic paging ability for `GET` requests for `components`.
 
+### Changed
+- Modified operators to use paging when requesting BOS components, using a page size equal to the `max_components_batch_size` option.
+- Put all requests code into context managers -- this includes the HTTP adapters, the sessions, and the request responses.
+
 ## [2.31.0] - 2024-11-01
 ### Removed
 - Moved BOS reporter to https://github.com/Cray-HPE/bos-reporter

src/bos/common/clients/endpoints/request_error_handler.py

@@ -42,6 +42,7 @@ class BaseRequestErrorHandler(ABC):
     """
     The abstract base class for request error handlers that will be used by an API endpoint.
     """
+
     @classmethod
     @abstractmethod
     def handle_exception(cls, err: Exception,
@@ -53,6 +54,7 @@ class RequestErrorHandler(BaseRequestErrorHandler):
     """
     The default request error handler used by API endpoints.
     """
+
     @classmethod
     def handle_api_response_error(cls, err: ApiResponseError,
                                   request_data: RequestData) -> NoReturn:

src/bos/common/tenant_utils.py

@@ -25,10 +25,12 @@
 import functools
 import logging
 import hashlib
+from typing import Optional
 
 import connexion
+import requests
 from requests.exceptions import HTTPError
-from bos.common.utils import exc_type_msg, requests_retry_session, PROTOCOL
+from bos.common.utils import exc_type_msg, retry_session_get, PROTOCOL
 
 LOGGER = logging.getLogger(__name__)
 
@@ -73,21 +75,19 @@ def get_tenant_aware_key(key, tenant):
     return f"{tenant_hash}-{key_hash}"
 
 
-def get_tenant_data(tenant, session=None):
-    if not session:
-        session = requests_retry_session()
+def get_tenant_data(tenant, session: Optional[requests.Session] = None):
     url = f"{TENANT_ENDPOINT}/{tenant}"
-    response = session.get(url)
-    try:
-        response.raise_for_status()
-    except HTTPError as e:
-        LOGGER.error("Failed getting tenant data from tapms: %s",
-                     exc_type_msg(e))
-        if response.status_code == 404:
-            raise InvalidTenantException(
-                f"Data not found for tenant {tenant}") from e
-        raise
-    return response.json()
+    with retry_session_get(url, session=session) as response:
+        try:
+            response.raise_for_status()
+        except HTTPError as e:
+            LOGGER.error("Failed getting tenant data from tapms: %s",
+                         exc_type_msg(e))
+            if response.status_code == 404:
+                raise InvalidTenantException(
+                    f"Data not found for tenant {tenant}") from e
+            raise
+        return response.json()
 
 
 def get_tenant_component_set(tenant: str) -> set:

src/bos/common/utils.py

@@ -28,7 +28,7 @@
 from functools import partial
 import re
 import traceback
-from typing import Iterator, List, Optional, Unpack
+from typing import Iterator, Optional, Unpack
 
 # Third party imports
 from dateutil.parser import parse
@@ -77,7 +77,6 @@ def duration_to_timedelta(timestamp: str):
     connect_timeout=3,
     read_timeout=10)
 
-
 retry_session_manager = partial(rrs.retry_session_manager,
                                 protocol=PROTOCOL,
                                 **DEFAULT_RETRY_ADAPTER_ARGS)
@@ -122,12 +121,6 @@ def retry_session_get(*get_args,
         return _session.get(*get_args, **get_kwargs)
 
 
-requests_retry_session = partial(rrs.requests_retry_session,
-                                 session=None,
-                                 protocol=PROTOCOL,
-                                 **DEFAULT_RETRY_ADAPTER_ARGS)
-
-
 def compact_response_text(response_text: str) -> str:
     """
     Often JSON is "pretty printed" in response text, which is undesirable for our logging.
@@ -201,7 +194,7 @@ def using_sbps_check_kernel_parameters(kernel_parameters: str) -> bool:
     return "root=sbps-s3" in kernel_parameters
 
 
-def components_by_id(components: List[dict]) -> dict:
+def components_by_id(components: list[dict]) -> dict:
     """
     Input:
     * components: a list containing individual components
@@ -215,7 +208,7 @@ def components_by_id(components: List[dict]) -> dict:
     return {component["id"]: component for component in components}
 
 
-def reverse_components_by_id(components_by_id_map: dict) -> List[dict]:
+def reverse_components_by_id(components_by_id_map: dict) -> list[dict]:
     """
     Input:
     components_by_id_map: a dictionary with the name of each component as the

src/bos/operators/base.py

@@ -62,6 +62,7 @@ class MissingSessionData(BaseOperatorException):
     desired state.
     """
 
+
 class ApiClients:
     """
     Context manager to provide API clients to BOS operators.
@@ -97,7 +98,6 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         return self._stack.__exit__(exc_type, exc_val, exc_tb)
 
 
-
 class BaseOperator(ABC):
     """
     An abstract class for all BOS operators.

src/bos/operators/power_off_forceful.py

@@ -50,10 +50,10 @@ def name(self):
     def filters(self):
         return [
             self.BOSQuery(enabled=True,
-                     status=','.join([
-                         Status.power_off_forcefully_called,
-                         Status.power_off_gracefully_called
-                     ])),
+                          status=','.join([
+                              Status.power_off_forcefully_called,
+                              Status.power_off_gracefully_called
+                          ])),
             TimeSinceLastAction(seconds=options.max_power_off_wait_time),
             self.HSMState(),
         ]

src/bos/operators/power_on.py

@@ -80,7 +80,9 @@ def _act(self, components: Union[List[dict], None]):
             raise Exception(
                 f"Error encountered setting BSS information: {e}") from e
         try:
-            self.client.cfs.components.set_cfs(components, enabled=False, clear_state=True)
+            self.client.cfs.components.set_cfs(components,
+                                               enabled=False,
+                                               clear_state=True)
         except Exception as e:
             raise Exception(
                 f"Error encountered setting CFS information: {e}") from e
@@ -148,10 +150,11 @@ def _set_bss(self, boot_artifacts, bos_sessions, retries=5):
         for key, nodes in boot_artifacts.items():
             kernel, kernel_parameters, initrd = key
             try:
-                resp = self.client.bss.boot_parameters.set_bss(node_set=nodes,
-                                   kernel_params=kernel_parameters,
-                                   kernel=kernel,
-                                   initrd=initrd)
+                resp = self.client.bss.boot_parameters.set_bss(
+                    node_set=nodes,
+                    kernel_params=kernel_parameters,
+                    kernel=kernel,
+                    initrd=initrd)
                 resp.raise_for_status()
             except HTTPError as err:
                 LOGGER.error(
@@ -237,7 +240,8 @@ def _tag_images(self, boot_artifacts: Dict[Tuple[str, str, str], Set[str]],
         my_components_by_id = components_by_id(components)
         for image in image_ids:
             try:
-                self.client.ims.images.tag_image(image, "set", "sbps-project", "true")
+                self.client.ims.images.tag_image(image, "set", "sbps-project",
+                                                 "true")
             except Exception as e:
                 components_to_update = []
                 for node in image_id_to_nodes[image]:

src/bos/operators/session_cleanup.py

@@ -28,7 +28,6 @@
 from bos.common.clients.bos.options import options
 from bos.operators.base import BaseOperator, main
 
-
 LOGGER = logging.getLogger(__name__)
 
 

src/bos/operators/session_setup.py

@@ -40,7 +40,6 @@
 from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory
 from bos.operators.utils.rootfs.factory import ProviderFactory
 
-
 LOGGER = logging.getLogger(__name__)
 
 
@@ -75,7 +74,8 @@ def _run(self) -> None:
         LOGGER.info('Found %d sessions that require action', len(sessions))
         inventory_cache = Inventory(self.client.hsm)
         for data in sessions:
-            session = Session(data, inventory_cache, self.client.bos, self.HSMState)
+            session = Session(data, inventory_cache, self.client.bos,
+                              self.HSMState)
             session.setup(self.max_batch_size)
 
     def _get_pending_sessions(self):
@@ -84,7 +84,8 @@ def _get_pending_sessions(self):
 
 class Session:
 
-    def __init__(self, data, inventory_cache, bos_client: BOSClient, hsm_state: Callable[...,HSMState]):
+    def __init__(self, data, inventory_cache, bos_client: BOSClient,
+                 hsm_state: Callable[..., HSMState]):
         self.session_data = data
         self.inventory = inventory_cache
         self.bos_client = bos_client

src/bos/operators/status.py

@@ -30,7 +30,6 @@
 from bos.operators.filters import DesiredBootStateIsOff, BootArtifactStatesMatch, \
     DesiredConfigurationIsNone, LastActionIs, TimeSinceLastAction
 
-
 LOGGER = logging.getLogger(__name__)
 
 

src/bos/server/controllers/v2/components.py

@@ -101,14 +101,15 @@ def get_v2_components_data(id_list=None,
 
     Allows filtering using a comma separated list of ids.
     """
-    tenant_components = None if tenant is None else get_tenant_component_set(tenant)
+    tenant_components = None if tenant is None else get_tenant_component_set(
+        tenant)
 
     if id_list is not None:
         id_set = set(id_list)
         if tenant_components is not None:
             id_set.intersection_update(tenant_components)
     else:
-         id_set = tenant_components
+        id_set = tenant_components
 
     # If id_set is not None but is empty, that means no components in the system
     # will match our filter, so we can return an empty list immediately.