Release 2.29.0 for CSM 1.6

CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.29.0] - 2024-10-01
+### Added
+- Run `pylint` during builds
+- When validating boot sets, if a boot artifact is missing from a manifest, include the manifest S3 URL
+  in the associated message.
+- Add check that at least one hardware-specifier field is non-empty to `validate_sanitize_boot_sets`
+- When validating boot sets (at session template patch/creation/validation or session creation),
+  attempt to validate that the specified architecture matches that of the corresponding IMS image.
+  Mismatches are fatal errors.
+
+### Changed
+- Marked PATCH session status endpoint to be ignored by the CLI.
+- Eliminate redundancies in API spec by defining `V2SessionNameOrEmpty` schema.
+- Refactor `validate_boot_sets` function into multiple functions to make it easier to understand.
+- During migration to this BOS version, for boot sets with no `arch` field, add the field with its
+  default value.
+
+### Fixed
+- When validating boot sets, check all boot sets for severe errors before returning only warnings
+
 ## [2.28.0] - 2024-09-11
 ### Added
 - Added `reject_nids` BOS option, to reject Sessions and Session Template which appear to reference NIDs.

Dockerfile

@@ -79,7 +79,7 @@ RUN --mount=type=secret,id=netrc,target=/root/.netrc \
     pip3 list --format freeze && \
     python3 -m convert_oas30_schemas /app/openapi.json /app/lib/bos/server/openapi.jsonschema && \
     cat /app/lib/bos/server/openapi.jsonschema
-
+RUN pylint /app/convert_oas/convert_oas.py || true
 
 # Base image
 FROM alpine-base AS base
@@ -120,11 +120,30 @@ RUN --mount=type=secret,id=netrc,target=/root/.netrc \
 FROM base AS testing
 WORKDIR /app
 COPY test-requirements.txt .
-RUN --mount=type=secret,id=netrc,target=/root/.netrc cd /app && \
+RUN --mount=type=secret,id=netrc,target=/root/.netrc \
+    cd /app && \
     pip3 install --no-cache-dir -r test-requirements.txt && \
     pip3 list --format freeze
 
 
+# Pylint reporting
+FROM base AS pylint-base
+COPY srclist.txt docker_pylint.sh /app/venv/
+WORKDIR /app
+RUN --mount=type=secret,id=netrc,target=/root/.netrc \
+    pip3 install --no-cache-dir pylint -c constraints.txt && \
+    pip3 list --format freeze
+
+# Pylint errors-only
+FROM pylint-base AS pylint-errors-only
+WORKDIR /app/venv
+CMD [ "./docker_pylint.sh", "--errors-only" ]
+
+# Pylint full
+FROM pylint-base AS pylint-full
+WORKDIR /app/venv
+CMD [ "./docker_pylint.sh", "--fail-under", "9" ]
+
 # Codestyle reporting
 FROM testing AS codestyle
 WORKDIR /app

Jenkinsfile.github

@@ -93,7 +93,12 @@ pipeline {
                     }
                     steps {
                         echo "Docker args are ${env.DOCKER_ARGS}"
-                        sh "make image"
+                        sh "make image_setup"
+                        sh "make image_build"
+                        sh "make image_build_pylint_errors"
+                        sh "make image_run_pylint_errors"
+                        sh "make image_build_pylint_full"
+                        sh "make image_run_pylint_full"
                     }
                 }
                 stage('Chart') {

Makefile

@@ -1,7 +1,7 @@
 #
 # MIT License
 #
-# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -55,6 +55,7 @@ RPTR_SOURCE_PATH := ${RPTR_BUILD_DIR}/SOURCES/${RPTR_SOURCE_NAME}.tar.bz2
 all : runbuildprep lint image chart rptr_rpm
 local: cms_meta_tools runbuildprep image chart_setup chart_package
 chart: chart_setup chart_package chart_test
+image: image_setup image_build image_build_pylint_errors image_run_pylint_errors image_build_pylint_full image_run_pylint_full
 rptr_rpm: rptr_rpm_package_source rptr_rpm_build_source rptr_rpm_build
 
 clone_input_files:
@@ -90,9 +91,25 @@ rptr_rpm_prepare:
 		cp $(RPTR_SPEC_FILE) $(RPTR_BUILD_DIR)/SPECS/
 		cat $(RPTR_SPEC_FILE) $(RPTR_BUILD_DIR)/SPECS/bos-reporter.spec
 
-image:
+image_setup:
+		# Create list of BOS Python source files, to be checked later by pylint
+		find src/bos -type f -name \*.py -print | sed 's#^src/#/app/lib/#' | tr '\n' ' ' | tee srclist.txt
+
+image_build:
 		docker build --pull ${DOCKER_ARGS} --tag '${NAME}:${DOCKER_VERSION}' .
 
+image_build_pylint_errors:
+		docker build --pull ${DOCKER_ARGS} --target pylint-errors-only --tag 'pylint-errors-only:${DOCKER_VERSION}' .
+
+image_run_pylint_errors:
+		docker run --rm 'pylint-errors-only:${DOCKER_VERSION}'
+
+image_build_pylint_full:
+		docker build --pull ${DOCKER_ARGS} --target pylint-full --tag 'pylint-full:${DOCKER_VERSION}' .
+
+image_run_pylint_full:
+		docker run --rm 'pylint-full:${DOCKER_VERSION}'
+
 chart_package:
 		helm dep up ${CHART_PATH}/${NAME}
 		helm package ${CHART_PATH}/${NAME} -d ${CHART_PATH}/.packaged --app-version ${DOCKER_VERSION} --version ${CHART_VERSION}

api/openapi.yaml.in

@@ -436,6 +436,10 @@ components:
       maxLength: 127
       pattern: '^[a-zA-Z0-9](?:[-._a-zA-Z0-9]{0,125}[a-zA-Z0-9])?$'
       example: "session-20190728032600"
+    V2SessionNameOrEmpty:
+      oneOf:
+        - $ref: '#/components/schemas/V2SessionName'
+        - $ref: '#/components/schemas/EmptyString'
     V2SessionOperation:
       type: string
       enum: ['boot', 'reboot', 'shutdown']
@@ -762,9 +766,7 @@ components:
         configuration:
           $ref: '#/components/schemas/CfsConfiguration'
         session:
-          oneOf:
-            - $ref: '#/components/schemas/V2SessionName'
-            - $ref: '#/components/schemas/EmptyString'
+          $ref: '#/components/schemas/V2SessionNameOrEmpty'
         last_updated:
           $ref: '#/components/schemas/V2ComponentLastUpdated'
       additionalProperties: false
@@ -851,9 +853,7 @@ components:
           description: A description of the most recent error to impact the Component.
           maxLength: 65536
         session:
-          oneOf:
-            - $ref: '#/components/schemas/V2SessionName'
-            - $ref: '#/components/schemas/EmptyString'
+          $ref: '#/components/schemas/V2SessionNameOrEmpty'
         retry_policy:
           type: integer
           description: |
@@ -892,9 +892,7 @@ components:
           description: A description of the most recent error to impact the Component.
           maxLength: 65536
         session:
-          oneOf:
-            - $ref: '#/components/schemas/V2SessionName'
-            - $ref: '#/components/schemas/EmptyString'
+          $ref: '#/components/schemas/V2SessionNameOrEmpty'
         retry_policy:
           type: integer
           description: |
@@ -1567,6 +1565,7 @@ paths:
       tags:
         - v2
         - sessions
+        - cli_ignore
       x-openapi-router-controller: bos.server.controllers.v2.sessions
       operationId: patch_v2_session
       requestBody:

src/__init__.py → docker_pylint.sh

@@ -1,7 +1,8 @@
+#!/usr/bin/env sh
 #
 # MIT License
 #
-# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP
+# (C) Copyright 2024 Hewlett Packard Enterprise Development LP
 #
 # Permission is hereby granted, free of charge, to any person obtaining a
 # copy of this software and associated documentation files (the "Software"),
@@ -21,3 +22,5 @@
 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
 #
+
+pylint "$@" $(cat ./srclist.txt)

src/bos/common/utils.py

@@ -150,14 +150,14 @@ def components_by_id(components: List[dict]) -> dict:
     """
     return { component["id"]: component for component in components }
 
-def reverse_components_by_id(components_by_id: dict) -> List[dict]:
+def reverse_components_by_id(components_by_id_map: dict) -> List[dict]:
     """
     Input:
-    components_by_id: a dictionary with the name of each component as the
+    components_by_id_map: a dictionary with the name of each component as the
     key and the value being the entire component itself.
     Return:
     A list with each component as an element
 
     Purpose: Reverse the effect of components_by_id.
     """
-    return list(components_by_id.values())
+    return list(components_by_id_map.values())

src/bos/operators/status.py

@@ -123,6 +123,7 @@ def _check_status(self, component, power_state, cfs_component):
         Calculate the component's current status based upon its power state and CFS configuration
         state. If its status differs from the status in the database, return this information.
         """
+        error = None
         if power_state and cfs_component:
             phase, override, disable, error, action_failed = self._calculate_status(component,
                                                                                     power_state,

src/bos/operators/utils/boot_image_metadata/__init__.py

@@ -66,6 +66,12 @@ def rootfs(self):
         """
         return None
 
+    @property
+    def arch(self):
+        """
+        Get the arch
+        """
+        return None
 
 class BootImageMetaDataBadRead(Exception):
     """

src/bos/operators/utils/boot_image_metadata/s3_boot_image_metadata.py

@@ -27,7 +27,9 @@
 
 from bos.common.utils import exc_type_msg
 from bos.operators.utils.boot_image_metadata import BootImageMetaData, BootImageMetaDataBadRead
-from bos.operators.utils.clients.s3 import S3BootArtifacts, S3MissingConfiguration, \
+from bos.operators.utils.clients.ims import get_image, get_ims_id_from_s3_url, ImageNotFound, \
+                                            DEFAULT_IMS_IMAGE_ARCH
+from bos.operators.utils.clients.s3 import S3BootArtifacts, S3MissingConfiguration, S3Url, \
                                            ArtifactNotFound
 
 LOGGER = logging.getLogger('bos.operators.utils.boot_image_metadata.s3_boot_image_metadata')
@@ -197,3 +199,50 @@ def boot_parameters_etag(self):
         if bp:
             return bp['link']['etag']
         return None
+
+    @property
+    def manifest_s3_url(self) -> S3Url:
+        """
+        Returns the S3 URL to the boot manifest
+        """
+        return self.boot_artifacts.s3url
+
+    @property
+    def arch(self):
+        """
+        Extract the IMS image ID from the S3 manifest path.
+        Query IMS to get the architecture of the image.
+        Return the 'arch' field from the IMS image
+        If image is not in IMS, or 'arch' field not set, log warnings and return None.
+        (since technically BOS does not require the images to be in IMS)
+        """
+        s3_url = self.manifest_s3_url
+        ims_id = get_ims_id_from_s3_url(s3_url)
+        if not ims_id:
+            LOGGER.warning(
+                "Boot artifact S3 URL '%s' does not follow the expected IMS image convention",
+                s3_url)
+            return None
+        try:
+            ims_image_data = get_image(ims_id)
+        except ImageNotFound:
+            LOGGER.warning(
+                "Can't determine architecture of '%s' because image '%s' does not exist in IMS",
+                s3_url.url, ims_id)
+            return None
+        except Exception as err:
+            LOGGER.error("Error getting IMS image data for '%s' (S3 path '%s'): %s", ims_id,
+                         s3_url.url, exc_type_msg(err))
+            return None
+        try:
+            return ims_image_data["arch"]
+        except KeyError:
+            LOGGER.warning("Defaulting to '%s' because 'arch' field not set in IMS image '%s' "
+                           "(s3 path '%s'): %s", DEFAULT_IMS_IMAGE_ARCH, ims_id, s3_url.url,
+                           ims_image_data)
+            return DEFAULT_IMS_IMAGE_ARCH
+        except Exception as err:
+            LOGGER.error("IMS image '%s' (s3 path '%s'): %s", ims_id, s3_url.url, ims_image_data)
+            LOGGER.error("Error getting 'arch' field for IMS image '%s': %s", ims_id,
+                         exc_type_msg(err))
+        return None