Merge pull request #255 from Cray-HPE/casmcms-8941-csm-1.5

CASMCMS-8941: Break up large CFS component queries to avoid failures

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.10.6] - 2024-03-07
+### Fixed
+- Break up large CFS component queries to avoid errors
+
 ## [2.10.5] - 2024-03-04
 ### Fixed
 - Corrected minor errors in a couple description fields in API spec (no functional impact)

src/bos/operators/filters/filters.py

@@ -2,7 +2,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -31,7 +31,7 @@
 from bos.common.utils import get_current_time, load_timestamp
 from bos.operators.filters.base import BaseFilter, DetailsFilter, IDFilter, LocalFilter
 from bos.operators.utils.clients.bos import BOSClient
-from bos.operators.utils.clients.cfs import get_components as get_cfs_components
+from bos.operators.utils.clients.cfs import get_components_from_id_list as get_cfs_components_from_id_list
 from bos.operators.utils.clients.hsm import get_components as get_hsm_components
 
 LOGGER = logging.getLogger('bos.operators.filters.filters')
@@ -198,8 +198,8 @@ def __init__(self):
         super().__init__()
 
     def _filter(self, components: List[dict]) -> List[dict]:
-        component_ids = ','.join([component['id'] for component in components])
-        cfs_components = get_cfs_components(ids=component_ids)
+        component_ids = [component['id'] for component in components]
+        cfs_components = get_cfs_components_from_id_list(id_list=component_ids)
         self.cfs_components_dict = {component['id']: component for component in cfs_components}
         matches = LocalFilter._filter(self, components)
         # Clear this, so there are no lingering side-effects of running this method.

src/bos/operators/utils/clients/cfs.py

@@ -1,7 +1,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -33,6 +33,7 @@
 
 LOGGER = logging.getLogger('bos.operators.utils.clients.cfs')
 
+GET_BATCH_SIZE = 200
 PATCH_BATCH_SIZE = 1000
 
 
@@ -59,6 +60,17 @@ def patch_components(data, session=None):
         raise
 
 
+def get_components_from_id_list(id_list):
+    session = requests_retry_session()
+    component_list = []
+    while id_list:
+        next_batch = id_list[:GET_BATCH_SIZE]
+        next_comps = get_components(session=session, ids=','.join(next_batch))
+        component_list.extend(next_comps)
+        id_list = id_list[GET_BATCH_SIZE:]
+    return component_list
+
+
 def patch_desired_config(node_ids, desired_config, enabled=False, tags=None, clear_state=False):
     session = requests_retry_session()
     data = []