CASMCMS-9225: Add paging to component list requests; use context managers for all requests

.gitignore

@@ -2,6 +2,8 @@ results
 .project
 .settings
 .version
+.venv
+.idea
 .pydevproject
 ansible/bos_deploy.retry
 .tox

CHANGELOG.md

@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Added basic paging ability for `GET` requests for `components`.
+
+### Changed
+- Modified operators to use paging when requesting BOS components, using a page size equal to the `max_components_batch_size` option.
+- Put all requests code into context managers -- this includes the HTTP adapters, the sessions, and the request responses.
 
 ## [2.31.0] - 2024-11-01
 ### Removed

api/openapi.yaml.in

@@ -1691,6 +1691,20 @@ paths:
           in: query
           description: |-
             Retrieve the Components with the given status.
+        - name: start_after_id
+          schema:
+            $ref: '#/components/schemas/V2ComponentId'
+          in: query
+          description: |-
+            Begin listing Components after the specified ID. Used for paging.
+        - name: page_size
+          schema:
+            type: integer
+            minimum: 0
+            maximum: 1048576
+          in: query
+          description: |-
+            Maximum number of Components to include in response. Used for paging.
       description: |-
         Retrieve the full collection of Components in the form of a
         ComponentArray. Full results can also be filtered by query

constraints.txt.in

@@ -37,7 +37,7 @@ PyYAML>=6.0.1,<6.1
 redis>=5.0,<5.1
 requests>=2.28.2,<2.29
 requests-oauthlib>=1.3.1,<1.4
-requests-retry-session>=0.1,<0.2
+requests-retry-session>=2.0,<2.1
 retrying>=1.3.4,<1.4
 rsa>=4.9,<4.10
 s3transfer>=0.6.2,<0.7

requirements.txt

@@ -15,7 +15,7 @@ python-dateutil
 PyYAML
 redis
 requests
-requests-retry-session
+requests-retry-session>=2.0
 urllib3
 
 # The purpose of this file is to contain python runtime requirements

src/bos/common/clients/api_client.py

@@ -0,0 +1,62 @@
+#
+# MIT License
+#
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+from abc import ABC
+from typing import Type, TypeVar
+
+from bos.common.clients.endpoints import BaseGenericEndpoint
+from bos.common.utils import RetrySessionManager
+
+ClientEndpoint = TypeVar('ClientEndpoint', bound=BaseGenericEndpoint)
+
+
+class APIClient(RetrySessionManager, ABC):
+    """
+    As a subclass of RetrySessionManager, this class can be used as a context manager,
+    and will have a requests session available as self.requests_session
+
+    This context manager is used to provide API endpoints, via subclassing.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._endpoint_values: dict[Type[ClientEndpoint], ClientEndpoint] = {}
+
+    def get_endpoint(self,
+                     endpoint_type: Type[ClientEndpoint]) -> ClientEndpoint:
+        """
+        Endpoints are created only as needed, and passed the manager retry session.
+        """
+        if endpoint_type not in self._endpoint_values:
+            self._endpoint_values[endpoint_type] = endpoint_type(
+                self.requests_session)
+        return self._endpoint_values[endpoint_type]
+
+    def __exit__(self, exc_type, exc_val, exc_tb) -> bool | None:
+        """
+        The only cleanup we need to do when exiting the context manager is to clear out
+        our list of API clients. Our call to super().__exit__ will take care of closing
+        out the underlying request session.
+        """
+        self._endpoint_values.clear()
+        return super().__exit__(exc_type, exc_val, exc_tb)

src/bos/common/clients/bos/__init__.py

@@ -0,0 +1,25 @@
+#
+# MIT License
+#
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+
+from .client import BOSClient

src/bos/common/clients/bos/base.py

@@ -0,0 +1,121 @@
+#
+# MIT License
+#
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+from abc import ABC
+import logging
+
+from bos.common.clients.endpoints import BaseEndpoint
+from bos.common.tenant_utils import get_new_tenant_header
+from bos.common.utils import PROTOCOL
+
+LOGGER = logging.getLogger(__name__)
+
+API_VERSION = 'v2'
+SERVICE_NAME = 'cray-bos'
+BASE_BOS_ENDPOINT = f"{PROTOCOL}://{SERVICE_NAME}/{API_VERSION}"
+
+
+class BaseBosEndpoint(BaseEndpoint, ABC):
+    """
+    This base class provides generic access to the BOS API.
+    The individual endpoint needs to be overridden for a specific endpoint.
+    """
+    BASE_ENDPOINT = BASE_BOS_ENDPOINT
+
+
+class BaseBosNonTenantAwareEndpoint(BaseBosEndpoint, ABC):
+    """
+    This base class provides generic access to the BOS API for non-tenant-aware endpoints
+    The individual endpoint needs to be overridden for a specific endpoint.
+    """
+
+    def get_item(self, item_id):
+        """Get information for a single BOS item"""
+        return self.get(uri=item_id)
+
+    def get_items(self, **kwargs):
+        """Get information for all BOS items"""
+        return self.get(params=kwargs)
+
+    def update_item(self, item_id, data):
+        """Update information for a single BOS item"""
+        return self.patch(uri=item_id, json=data)
+
+    def update_items(self, data):
+        """Update information for multiple BOS items"""
+        return self.patch(json=data)
+
+    def put_items(self, data):
+        """Put information for multiple BOS Items"""
+        return self.put(json=data)
+
+    def delete_items(self, **kwargs):
+        """Delete information for multiple BOS items"""
+        return self.delete(params=kwargs)
+
+
+class BaseBosTenantAwareEndpoint(BaseBosEndpoint, ABC):
+    """
+    This base class provides generic access to the BOS API for tenant aware endpoints.
+    The individual endpoint needs to be overridden for a specific endpoint.
+    """
+
+    def get_item(self, item_id, tenant):
+        """Get information for a single BOS item"""
+        return self.get(uri=item_id, headers=get_new_tenant_header(tenant))
+
+    def get_items(self, **kwargs):
+        """Get information for all BOS items"""
+        headers = None
+        if "tenant" in kwargs:
+            tenant = kwargs.pop("tenant")
+            headers = get_new_tenant_header(tenant)
+        return self.get(params=kwargs, headers=headers)
+
+    def update_item(self, item_id, tenant, data):
+        """Update information for a single BOS item"""
+        return self.patch(uri=item_id,
+                          json=data,
+                          headers=get_new_tenant_header(tenant))
+
+    def update_items(self, tenant, data):
+        """Update information for multiple BOS items"""
+        return self.patch(json=data, headers=get_new_tenant_header(tenant))
+
+    def post_item(self, item_id, tenant, data=None):
+        """Post information for a single BOS item"""
+        return self.post(uri=item_id,
+                         json=data,
+                         headers=get_new_tenant_header(tenant))
+
+    def put_items(self, tenant, data):
+        """Put information for multiple BOS items"""
+        return self.put(json=data, headers=get_new_tenant_header(tenant))
+
+    def delete_items(self, **kwargs):
+        """Delete information for multiple BOS items"""
+        headers = None
+        if "tenant" in kwargs:
+            tenant = kwargs.pop("tenant")
+            headers = get_new_tenant_header(tenant)
+        return self.delete(params=kwargs, headers=headers)

src/bos/common/clients/bos/client.py

@@ -0,0 +1,43 @@
+#
+# MIT License
+#
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+from bos.common.clients.api_client import APIClient
+
+from .components import ComponentEndpoint
+from .sessions import SessionEndpoint
+from .session_templates import SessionTemplateEndpoint
+
+
+class BOSClient(APIClient):
+
+    @property
+    def components(self) -> ComponentEndpoint:
+        return self.get_endpoint(ComponentEndpoint)
+
+    @property
+    def sessions(self) -> SessionEndpoint:
+        return self.get_endpoint(SessionEndpoint)
+
+    @property
+    def session_templates(self) -> SessionTemplateEndpoint:
+        return self.get_endpoint(SessionTemplateEndpoint)

src/bos/operators/utils/clients/bos/components.py → src/bos/common/clients/bos/components.py

@@ -23,19 +23,31 @@
 #
 import logging
 
-from .base import BaseBosEndpoint
+from .base import BaseBosNonTenantAwareEndpoint
+from .options import options
 
-LOGGER = logging.getLogger('bos.operators.utils.clients.bos.components')
+LOGGER = logging.getLogger(__name__)
 
 
-class ComponentEndpoint(BaseBosEndpoint):
+class ComponentEndpoint(BaseBosNonTenantAwareEndpoint):
     ENDPOINT = __name__.lower().rsplit('.', maxsplit=1)[-1]
 
     def get_component(self, component_id):
         return self.get_item(component_id)
 
     def get_components(self, **kwargs):
-        return self.get_items(**kwargs)
+        page_size = options.max_component_batch_size
+        if page_size == 0:
+            return self.get_items(**kwargs)
+        next_page = self.get_items(page_size=page_size, **kwargs)
+        results = next_page
+        while len(next_page) == page_size:
+            last_id = next_page[-1]["id"]
+            next_page = self.get_items(page_size=page_size,
+                                       start_after_id=last_id,
+                                       **kwargs)
+            results.extend(next_page)
+        return results
 
     def update_component(self, component_id, data):
         return self.update_item(component_id, data)

src/bos/operators/utils/clients/bos/options.py → src/bos/common/clients/bos/options.py

@@ -23,17 +23,20 @@
 #
 import logging
 import json
+from typing import Optional
 
-from requests.exceptions import HTTPError, ConnectionError
+import requests
+from requests.exceptions import HTTPError
+from requests.exceptions import ConnectionError as RequestsConnectionError
 from urllib3.exceptions import MaxRetryError
 
 from bos.common.options import OptionsCache
-from bos.common.utils import exc_type_msg, requests_retry_session
-from bos.operators.utils.clients.bos.base import BASE_ENDPOINT
+from bos.common.utils import exc_type_msg, retry_session_get
+from bos.common.clients.bos.base import BASE_BOS_ENDPOINT
 
-LOGGER = logging.getLogger('bos.operators.utils.clients.bos.options')
+LOGGER = logging.getLogger(__name__)
 __name = __name__.lower().rsplit('.', maxsplit=1)[-1]
-ENDPOINT = f"{BASE_ENDPOINT}/{__name}"
+ENDPOINT = f"{BASE_BOS_ENDPOINT}/{__name}"
 
 
 class Options(OptionsCache):
@@ -43,15 +46,15 @@ class Options(OptionsCache):
     This caches the options so that frequent use of these options do not all
     result in network calls.
     """
-    def _get_options(self) -> dict:
+
+    def _get_options(self, session: Optional[requests.Session] = None) -> dict:
         """Retrieves the current options from the BOS api"""
-        session = requests_retry_session()
         LOGGER.debug("GET %s", ENDPOINT)
         try:
-            response = session.get(ENDPOINT)
-            response.raise_for_status()
-            return json.loads(response.text)
-        except (ConnectionError, MaxRetryError) as e:
+            with retry_session_get(ENDPOINT, session=session) as response:
+                response.raise_for_status()
+                return json.loads(response.text)
+        except (RequestsConnectionError, MaxRetryError) as e:
             LOGGER.error("Unable to connect to BOS: %s", exc_type_msg(e))
         except HTTPError as e:
             LOGGER.error("Unexpected response from BOS: %s", exc_type_msg(e))