From a7806b762b24cd58ef281d3afd678b640838602f Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 7 Oct 2024 11:43:18 -0400 Subject: [PATCH 1/2] CASMCMS-9146: Add BOS option ims_images_must_exist --- CHANGELOG.md | 8 +- api/openapi.yaml.in | 9 ++ src/bos/common/options.py | 5 + src/bos/server/controllers/v2/boot_set.py | 143 ++++++++++++---------- 4 files changed, 97 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d278cd5..c88c7a69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Added `ims_errors_fatal` BOS option. This determines whether or not an IMS failure +#### BOS options +- `ims_errors_fatal`: This determines whether or not an IMS failure is considered fatal even when BOS could continue despite the failure. Specifically, this comes up when validating image architecture in a boot set. By default this is false. Note that this has no effect for boot sets that: - Have non-IMS images - Have IMS images but the image does not exist in IMS - Have `Other` architecture +- `ims_images_must_exist`: This determines whether or not BOS considers it a fatal error if + a boot set has an IMS boot image which does not exist in IMS. If false (the default), then + BOS will only log warnings about these. If true, then these will cause boot set validations + to fail. Note that if `ims_images_must_exist` is true but `ims_errors_fatal` is false, then + a failure to determine whether or not an image is in IMS will NOT result in a fatal error. ### Changed - Refactored some BOS Options code to use abstract base classes, to avoid code duplication. diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index d72885a4..9cb652be 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1025,6 +1025,15 @@ components: In the above situation, if this option is true, then the validation will fail. Otherwise, if the option is false, then a warning will be logged, but the validation will not be failed because of this. + ims_images_must_exist: + type: boolean + description: | + This option modifies how BOS behaves when validating a boot set whose boot image appears to be from IMS. + Specifically, this option comes into play when the image does not actually exist in IMS. + In the above situation, if this option is true, then the validation will fail. + Otherwise, if the option is false, then a warning will be logged, but the validation will not + be failed because of this. Note that if ims_images_must_exist is true but ims_errors_fatal is false, then + a failure to determine whether or not an image is in IMS will NOT result in a fatal error. logging_level: type: string description: The logging level for all BOS services diff --git a/src/bos/common/options.py b/src/bos/common/options.py index f8bbfbed..d876e50a 100644 --- a/src/bos/common/options.py +++ b/src/bos/common/options.py @@ -36,6 +36,7 @@ 'disable_components_on_completion': True, 'discovery_frequency': 300, 'ims_errors_fatal': False, + 'ims_images_must_exist': False, 'logging_level': 'INFO', 'max_boot_wait_time': 1200, 'max_component_batch_size': 2800, @@ -89,6 +90,10 @@ def discovery_frequency(self) -> int: def ims_errors_fatal(self) -> bool: return bool(self.get_option('ims_errors_fatal')) + @property + def ims_images_must_exist(self) -> bool: + return bool(self.get_option('ims_images_must_exist')) + @property def logging_level(self) -> str: return str(self.get_option('logging_level')) diff --git a/src/bos/server/controllers/v2/boot_set.py b/src/bos/server/controllers/v2/boot_set.py index a554e097..a04c9911 100644 --- a/src/bos/server/controllers/v2/boot_set.py +++ b/src/bos/server/controllers/v2/boot_set.py @@ -25,11 +25,10 @@ from functools import partial import logging from bos.common.utils import exc_type_msg, requests_retry_session -from bos.operators.utils.boot_image_metadata import BootImageMetaData from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory from bos.operators.utils.clients.ims import get_arch_from_image_data, get_image, \ get_ims_id_from_s3_url, ImageNotFound -from bos.operators.utils.clients.s3 import S3Object, ArtifactNotFound +from bos.operators.utils.clients.s3 import S3Object, S3Url, ArtifactNotFound from bos.server.controllers.v2.options import OptionsData from bos.server.utils import canonize_xname, ParsingException @@ -157,65 +156,30 @@ def _validate_boot_set(bs: dict, operation: str, options_data: OptionsData) -> l for field in HARDWARE_SPECIFIER_FIELDS] if not any(specified): raise BootSetError(f"No non-empty hardware specifier field {HARDWARE_SPECIFIER_FIELDS}") - try: - if any(node[:3] == "nid" for node in bs["node_list"]): - msg = "Has NID in 'node_list'" - if options_data.reject_nids: - raise BootSetError(msg) - # Otherwise, log this as a warning -- even if reject_nids is not set, - # BOS still doesn't support NIDs, so this is still undesirable - warning_msgs.append(msg) - except KeyError: - # If there is no node_list field, not a problem - pass + + if "node_list" in bs and any(node[:3] == "nid" for node in bs["node_list"]): + msg = "Has NID in 'node_list'" + if options_data.reject_nids: + raise BootSetError(msg) + # Otherwise, log this as a warning -- even if reject_nids is not set, + # BOS still doesn't support NIDs, so this is still undesirable + warning_msgs.append(msg) if operation in ['boot', 'reboot']: - # Verify that the boot artifacts exist try: - image_metadata = BootImageMetaDataFactory(bs)() - except Exception as err: - raise BootSetError(f"Can't find boot artifacts. Error: {exc_type_msg(err)}") from err + validate_boot_artifacts(bs) + except BootSetWarning as err: + warning_msgs.append(str(err)) try: - validate_ims_boot_image(bs, image_metadata, options_data) + validate_ims_boot_image(bs, options_data) except BootSetWarning as err: warning_msgs.append(str(err)) - # Check boot artifacts' S3 headers - for boot_artifact in ["kernel"]: - try: - artifact = getattr(image_metadata.boot_artifacts, boot_artifact) - path = artifact ['link']['path'] - etag = artifact['link']['etag'] - obj = S3Object(path, etag) - _ = obj.object_header - except Exception as err: - raise BootSetError(f"Can't find {boot_artifact} in " - f"{image_metadata.manifest_s3_url.url}. " - f"Error: {exc_type_msg(err)}") from err - - for boot_artifact in ["initrd", "boot_parameters"]: - try: - artifact = getattr(image_metadata.boot_artifacts, boot_artifact) - if not artifact: - raise ArtifactNotFound() - path = artifact ['link']['path'] - etag = artifact['link']['etag'] - obj = S3Object(path, etag) - _ = obj.object_header - except ArtifactNotFound as err: - warning_msgs.append( - f"{image_metadata.manifest_s3_url.url} doesn't contain a {boot_artifact}") - except Exception as err: - warning_msgs.append(f"Can't find {boot_artifact} in " - f"{image_metadata.manifest_s3_url.url}. " - f"Warning: {exc_type_msg(err)}") - return warning_msgs -def validate_ims_boot_image(bs: dict, image_metadata: BootImageMetaData, - options_data: OptionsData) -> None: +def validate_ims_boot_image(bs: dict, options_data: OptionsData) -> None: """ If the boot set architecture is not set to Other, check that the IMS image architecture matches the boot set architecture (treating a boot set architecture @@ -223,9 +187,14 @@ def validate_ims_boot_image(bs: dict, image_metadata: BootImageMetaData, Otherwise, at least validate whether the boot image is in IMS, if we expect it to be. """ + try: + bs_path = bs["path"] + except KeyError as err: + raise BootSetError("Missing required 'path' field") from err + bs_arch = bs.get("arch", DEFAULT_ARCH) - ims_id = get_ims_image_id(image_metadata) + ims_id = get_ims_image_id(bs_path) # If IMS being inaccessible is not a fatal error, then reduce the number # of retries we make, to prevent a lengthy delay @@ -234,10 +203,12 @@ def validate_ims_boot_image(bs: dict, image_metadata: BootImageMetaData, try: image_data = get_ims_image_data(ims_id, num_retries) except ImageNotFound as err: + if options_data.ims_images_must_exist: + raise BootSetError(str(err)) from err raise BootSetWarning(str(err)) from err except Exception as err: if options_data.ims_errors_fatal: - raise err + raise BootSetError(exc_type_msg(err)) from err if bs_arch != 'Other': # This means that this error is preventing us from validating the # boot set architecture @@ -254,7 +225,7 @@ def validate_ims_boot_image(bs: dict, image_metadata: BootImageMetaData, except Exception as err: # This most likely indicates that the IMS image data we got wasn't even a dict if options_data.ims_errors_fatal: - raise err + raise BootSetError(exc_type_msg(err)) from err raise BootSetWarning(str(err)) from err if EXPECTED_IMS_ARCH[bs_arch] != ims_image_arch: @@ -305,20 +276,20 @@ def validate_sanitize_boot_set(bs_name: str, bs_data: dict, options_data: Option raise ParsingException(f"boot_sets key ({bs_name}) does not match 'name' " f"field of corresponding boot set ({bs_data['name']})") - # Get the boot image metadata + # Check the boot artifacts try: - image_metadata = BootImageMetaDataFactory(bs_data)() + validate_boot_artifacts(bs_data) + except (BootSetError, BootSetWarning) as err: + LOGGER.warning(str(err)) + + # Validate the boot set IMS image + try: + validate_ims_boot_image(bs_data, options_data) + except BootSetWarning as err: + LOGGER.warning("Boot set '%s': %s", bs_name, err) + LOGGER.warning('Boot set contents: %s', bs_data) except Exception as err: - LOGGER.warning("Can't find boot artifacts: %s", exc_type_msg(err)) - else: # No exception was raised in try block - # Validate the boot set IMS image - try: - validate_ims_boot_image(bs_data, image_metadata, options_data) - except BootSetWarning as err: - LOGGER.warning("Boot set '%s': %s", bs_name, err) - LOGGER.warning('Boot set contents: %s', bs_data) - except Exception as err: - raise ParsingException(str(err)) from err + raise ParsingException(exc_type_msg(err)) from err # Validate that the boot set has at least one of the HARDWARE_SPECIFIER_FIELDS if not any(field_name in bs_data for field_name in HARDWARE_SPECIFIER_FIELDS): @@ -359,14 +330,14 @@ def validate_sanitize_boot_set(bs_name: str, bs_data: dict, options_data: Option bs_data["node_list"] = new_node_list -def get_ims_image_id(image_metadata: BootImageMetaData) -> str: +def get_ims_image_id(path: str) -> str: """ If the image is an IMS image, return its ID. Raise NonImsImage otherwise, Note that this does not actually check IMS to see if the ID exists. """ - s3_url = image_metadata.manifest_s3_url + s3_url = S3Url(path) ims_id = get_ims_id_from_s3_url(s3_url) if ims_id: return ims_id @@ -385,3 +356,41 @@ def get_ims_image_data(ims_id: str, num_retries: int|None=None) -> dict: # https://github.com/pylint-dev/pylint/issues/2271 kwargs['session'] = requests_retry_session(retries=4) # pylint: disable=redundant-keyword-arg return get_image(**kwargs) + + +def validate_boot_artifacts(bs: dict): + # Verify that the boot artifacts exist + try: + image_metadata = BootImageMetaDataFactory(bs)() + except Exception as err: + raise BootSetError(f"Can't find boot artifacts. Error: {exc_type_msg(err)}") from err + + # Check boot artifacts' S3 headers + for boot_artifact in ["kernel"]: + try: + artifact = getattr(image_metadata.boot_artifacts, boot_artifact) + path = artifact ['link']['path'] + etag = artifact['link']['etag'] + obj = S3Object(path, etag) + _ = obj.object_header + except Exception as err: + raise BootSetError(f"Can't find {boot_artifact} in " + f"{image_metadata.manifest_s3_url.url}. " + f"Error: {exc_type_msg(err)}") from err + + for boot_artifact in ["initrd", "boot_parameters"]: + try: + artifact = getattr(image_metadata.boot_artifacts, boot_artifact) + if not artifact: + raise ArtifactNotFound() + path = artifact ['link']['path'] + etag = artifact['link']['etag'] + obj = S3Object(path, etag) + _ = obj.object_header + except ArtifactNotFound as err: + raise BootSetWarning( + f"{image_metadata.manifest_s3_url.url} doesn't contain a {boot_artifact}") from err + except Exception as err: + raise BootSetWarning( + f"Unable to check {boot_artifact} in {image_metadata.manifest_s3_url.url}. " + f"Warning: {exc_type_msg(err)}") From 3a5f8939da97ec7ac00831054221d1f709178035 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 7 Oct 2024 11:59:50 -0400 Subject: [PATCH 2/2] Refactor boot_set.py into submodule for clarity --- CHANGELOG.md | 1 + src/bos/server/controllers/v2/boot_set.py | 396 ------------------ .../controllers/v2/boot_set/__init__.py | 29 ++ .../controllers/v2/boot_set/artifacts.py | 67 +++ .../server/controllers/v2/boot_set/defs.py | 41 ++ .../controllers/v2/boot_set/exceptions.py | 51 +++ src/bos/server/controllers/v2/boot_set/ims.py | 123 ++++++ .../controllers/v2/boot_set/sanitize.py | 117 ++++++ .../controllers/v2/boot_set/validate.py | 150 +++++++ src/bos/server/controllers/v2/sessions.py | 5 +- 10 files changed, 582 insertions(+), 398 deletions(-) delete mode 100644 src/bos/server/controllers/v2/boot_set.py create mode 100644 src/bos/server/controllers/v2/boot_set/__init__.py create mode 100644 src/bos/server/controllers/v2/boot_set/artifacts.py create mode 100644 src/bos/server/controllers/v2/boot_set/defs.py create mode 100644 src/bos/server/controllers/v2/boot_set/exceptions.py create mode 100644 src/bos/server/controllers/v2/boot_set/ims.py create mode 100644 src/bos/server/controllers/v2/boot_set/sanitize.py create mode 100644 src/bos/server/controllers/v2/boot_set/validate.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c88c7a69..14e30613 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Refactored some BOS Options code to use abstract base classes, to avoid code duplication. - Alphabetized options in API spec +- Refactored `controllers/v2/boot_sets.py` into its own module, for clarity ## [2.29.0] - 2024-10-01 ### Added diff --git a/src/bos/server/controllers/v2/boot_set.py b/src/bos/server/controllers/v2/boot_set.py deleted file mode 100644 index a04c9911..00000000 --- a/src/bos/server/controllers/v2/boot_set.py +++ /dev/null @@ -1,396 +0,0 @@ -# -# MIT License -# -# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR -# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -# OTHER DEALINGS IN THE SOFTWARE. -# - -from functools import partial -import logging -from bos.common.utils import exc_type_msg, requests_retry_session -from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory -from bos.operators.utils.clients.ims import get_arch_from_image_data, get_image, \ - get_ims_id_from_s3_url, ImageNotFound -from bos.operators.utils.clients.s3 import S3Object, S3Url, ArtifactNotFound -from bos.server.controllers.v2.options import OptionsData -from bos.server.utils import canonize_xname, ParsingException - -LOGGER = logging.getLogger('bos.server.controllers.v2.boot_set') - -BOOT_SET_SUCCESS = 0 -BOOT_SET_WARNING = 1 -BOOT_SET_ERROR = 2 - -# Valid boot sets are required to have at least one of these fields -HARDWARE_SPECIFIER_FIELDS = ( "node_list", "node_roles_groups", "node_groups" ) - -DEFAULT_ARCH = "X86" - -# Mapping from BOS boot set arch values to expected IMS image arch values -# Omits BOS Other value, since there is no corresponding IMS image arch value -EXPECTED_IMS_ARCH = { - "ARM": "aarch64", - "Unknown": "x86_64", - "X86": "x86_64" -} - - -class BootSetError(Exception): - """ - Generic error class for fatal problems found during boot set validation - """ - - -class BootSetWarning(Exception): - """ - Generic error class for non-fatal problems found during boot set validation - """ - - -class NonImsImage(BootSetWarning): - """ - Raised to indicate the boot set boot image is not from IMS - """ - - -class BootSetArchMismatch(BootSetError): - def __init__(self, bs_arch: str, expected_ims_arch: str, actual_ims_arch: str): - super().__init__(f"Boot set arch '{bs_arch}' means IMS image arch should be " - f"'{expected_ims_arch}', but actual IMS image arch is '{actual_ims_arch}'") - - -class CannotValidateBootSetArch(BootSetWarning): - def __init__(self, msg: str): - super().__init__(f"Can't validate boot image arch: {msg}") - - -def validate_boot_sets(session_template: dict, - operation: str, - template_name: str, - options_data: OptionsData|None=None) -> tuple[str, int]: - """ - Validates the boot sets listed in a session template. - It ensures that there are boot sets. - It checks that each boot set specifies nodes via at least one of the specifier fields. - Ensures that the boot artifacts exist. - - Inputs: - session_template (dict): Session template data - operation (str): Requested operation - template_name (str): The name of the session template; Note, during Session template - creation, the name in the session template data does not have - to match the name used to create the session template. - Returns: - Returns an error_code and a message - error_code: - 0 -- Success - 1 -- Warning, not fatal - 2 -- Error, fatal - """ - # Verify boot sets exist. - if not session_template.get('boot_sets', None): - msg = f"Session template '{template_name}' requires at least 1 boot set." - return BOOT_SET_ERROR, msg - - if options_data is None: - options_data = OptionsData() - - warning_msgs = [] - for bs_name, bs in session_template['boot_sets'].items(): - bs_msg = partial(_bs_msg, template_name=template_name, bs_name=bs_name) - try: - bs_warning_msgs = _validate_boot_set(bs=bs, operation=operation, - options_data=options_data) - except BootSetError as err: - msg = bs_msg(str(err)) - LOGGER.error(msg) - return BOOT_SET_ERROR, msg - except Exception as err: - LOGGER.error( - bs_msg(f"Unexpected exception in _validate_boot_set: {exc_type_msg(err)}")) - raise - for msg in map(bs_msg, bs_warning_msgs): - LOGGER.warning(msg) - warning_msgs.append(msg) - - if warning_msgs: - return BOOT_SET_WARNING, "; ".join(warning_msgs) - - return BOOT_SET_SUCCESS, "Valid" - - -def _bs_msg(msg: str, template_name: str, bs_name: str) -> str: - """ - Shortcut for creating validation error/warning messages for a specific bootset - """ - return f"Session template: '{template_name}' boot set: '{bs_name}': {msg}" - - -def _validate_boot_set(bs: dict, operation: str, options_data: OptionsData) -> list[str]: - """ - Helper function for validate_boot_sets that performs validation on a single boot set. - Raises BootSetError if fatal errors found. - Returns a list of warning messages (if any) - """ - warning_msgs = [] - - # Verify that the hardware is specified - specified = [bs.get(field, None) - for field in HARDWARE_SPECIFIER_FIELDS] - if not any(specified): - raise BootSetError(f"No non-empty hardware specifier field {HARDWARE_SPECIFIER_FIELDS}") - - if "node_list" in bs and any(node[:3] == "nid" for node in bs["node_list"]): - msg = "Has NID in 'node_list'" - if options_data.reject_nids: - raise BootSetError(msg) - # Otherwise, log this as a warning -- even if reject_nids is not set, - # BOS still doesn't support NIDs, so this is still undesirable - warning_msgs.append(msg) - - if operation in ['boot', 'reboot']: - try: - validate_boot_artifacts(bs) - except BootSetWarning as err: - warning_msgs.append(str(err)) - - try: - validate_ims_boot_image(bs, options_data) - except BootSetWarning as err: - warning_msgs.append(str(err)) - - return warning_msgs - - -def validate_ims_boot_image(bs: dict, options_data: OptionsData) -> None: - """ - If the boot set architecture is not set to Other, check that the IMS image - architecture matches the boot set architecture (treating a boot set architecture - of Unknown as X86) - - Otherwise, at least validate whether the boot image is in IMS, if we expect it to be. - """ - try: - bs_path = bs["path"] - except KeyError as err: - raise BootSetError("Missing required 'path' field") from err - - bs_arch = bs.get("arch", DEFAULT_ARCH) - - ims_id = get_ims_image_id(bs_path) - - # If IMS being inaccessible is not a fatal error, then reduce the number - # of retries we make, to prevent a lengthy delay - num_retries = None if options_data.ims_errors_fatal else 4 - - try: - image_data = get_ims_image_data(ims_id, num_retries) - except ImageNotFound as err: - if options_data.ims_images_must_exist: - raise BootSetError(str(err)) from err - raise BootSetWarning(str(err)) from err - except Exception as err: - if options_data.ims_errors_fatal: - raise BootSetError(exc_type_msg(err)) from err - if bs_arch != 'Other': - # This means that this error is preventing us from validating the - # boot set architecture - raise CannotValidateBootSetArch(str(err)) from err - # We weren't going to be validating the architecture, since it is Other, - # but we should still log this as a warning - raise BootSetWarning(str(err)) from err - - if bs_arch == 'Other': - raise CannotValidateBootSetArch("Boot set arch set to 'Other'") - - try: - ims_image_arch = get_arch_from_image_data(image_data) - except Exception as err: - # This most likely indicates that the IMS image data we got wasn't even a dict - if options_data.ims_errors_fatal: - raise BootSetError(exc_type_msg(err)) from err - raise BootSetWarning(str(err)) from err - - if EXPECTED_IMS_ARCH[bs_arch] != ims_image_arch: - raise BootSetArchMismatch(bs_arch=bs_arch, expected_ims_arch=EXPECTED_IMS_ARCH[bs_arch], - actual_ims_arch=ims_image_arch) - - -def validate_sanitize_boot_sets(template_data: dict, options_data: OptionsData|None=None) -> None: - """ - Calls validate_sanitize_boot_set on every boot set in the template. - Raises an exception if there are problems. - """ - # The boot_sets field is required. - try: - boot_sets = template_data["boot_sets"] - except KeyError as exc: - raise ParsingException("Missing required 'boot_sets' field") from exc - - # The boot_sets field must map to a dict - if not isinstance(boot_sets, dict): - raise ParsingException("'boot_sets' field has invalid type") - - # The boot_sets field must be non-empty - if not boot_sets: - raise ParsingException("Session templates must contain at least one boot set") - - if options_data is None: - options_data = OptionsData() - - # Finally, call validate_sanitize_boot_set on each boot set - for bs_name, bs in boot_sets.items(): - validate_sanitize_boot_set(bs_name, bs, options_data=options_data) - - -def validate_sanitize_boot_set(bs_name: str, bs_data: dict, options_data: OptionsData) -> None: - """ - Called when creating/updating a BOS session template. - Validates the boot set, and sanitizes it (editing it in place). - Raises ParsingException on error. - """ - if "name" not in bs_data: - # Set the field here -- this allows the name to be validated - # per the schema later - bs_data["name"] = bs_name - elif bs_data["name"] != bs_name: - # All keys in the boot_sets mapping must match the 'name' fields in the - # boot sets to which they map (if they contain a 'name' field). - raise ParsingException(f"boot_sets key ({bs_name}) does not match 'name' " - f"field of corresponding boot set ({bs_data['name']})") - - # Check the boot artifacts - try: - validate_boot_artifacts(bs_data) - except (BootSetError, BootSetWarning) as err: - LOGGER.warning(str(err)) - - # Validate the boot set IMS image - try: - validate_ims_boot_image(bs_data, options_data) - except BootSetWarning as err: - LOGGER.warning("Boot set '%s': %s", bs_name, err) - LOGGER.warning('Boot set contents: %s', bs_data) - except Exception as err: - raise ParsingException(exc_type_msg(err)) from err - - # Validate that the boot set has at least one of the HARDWARE_SPECIFIER_FIELDS - if not any(field_name in bs_data for field_name in HARDWARE_SPECIFIER_FIELDS): - raise ParsingException(f"Boot set {bs_name} has none of the following " - f"fields: {HARDWARE_SPECIFIER_FIELDS}") - - # Validate that at least one of the HARDWARE_SPECIFIER_FIELDS is non-empty - if not any(field_name in bs_data and bs_data[field_name] - for field_name in HARDWARE_SPECIFIER_FIELDS): - raise ParsingException(f"Boot set {bs_name} has no non-empty hardware-specifier fields: " - f"{HARDWARE_SPECIFIER_FIELDS}") - - # Last thing to do is validate/sanitize the node_list field, if it is present - try: - node_list = bs_data["node_list"] - except KeyError: - return - - # Make sure it is a list - if not isinstance(node_list, list): - raise ParsingException(f"Boot set {bs_name} has 'node_list' of invalid type") - - new_node_list = [] - for node in node_list: - # Make sure it is a list of strings - if not isinstance(node, str): - raise ParsingException(f"Boot set {bs_name} 'node_list' contains non-string element") - - # If reject_nids is set, raise an exception if any member of the node list - # begins with 'nid' - if options_data.reject_nids and node[:3] == 'nid': - raise ParsingException(f"reject_nids: Boot set {bs_name} 'node_list' contains a NID") - - # Canonize the xname and append it to the node list - new_node_list.append(canonize_xname(node)) - - # Update the node_list value in the boot set with the canonized version - bs_data["node_list"] = new_node_list - - -def get_ims_image_id(path: str) -> str: - """ - If the image is an IMS image, return its ID. - Raise NonImsImage otherwise, - Note that this does not actually check IMS to see if the ID - exists. - """ - s3_url = S3Url(path) - ims_id = get_ims_id_from_s3_url(s3_url) - if ims_id: - return ims_id - raise NonImsImage(f"Boot artifact S3 URL '{s3_url.url}' doesn't follow convention " - "for IMS images") - - -def get_ims_image_data(ims_id: str, num_retries: int|None=None) -> dict: - """ - Query IMS to get the image data and return it, - or raise an exception. - """ - kwargs = { "image_id": ims_id } - if num_retries is not None: - # A pylint bug generates a false positive error for this call - # https://github.com/pylint-dev/pylint/issues/2271 - kwargs['session'] = requests_retry_session(retries=4) # pylint: disable=redundant-keyword-arg - return get_image(**kwargs) - - -def validate_boot_artifacts(bs: dict): - # Verify that the boot artifacts exist - try: - image_metadata = BootImageMetaDataFactory(bs)() - except Exception as err: - raise BootSetError(f"Can't find boot artifacts. Error: {exc_type_msg(err)}") from err - - # Check boot artifacts' S3 headers - for boot_artifact in ["kernel"]: - try: - artifact = getattr(image_metadata.boot_artifacts, boot_artifact) - path = artifact ['link']['path'] - etag = artifact['link']['etag'] - obj = S3Object(path, etag) - _ = obj.object_header - except Exception as err: - raise BootSetError(f"Can't find {boot_artifact} in " - f"{image_metadata.manifest_s3_url.url}. " - f"Error: {exc_type_msg(err)}") from err - - for boot_artifact in ["initrd", "boot_parameters"]: - try: - artifact = getattr(image_metadata.boot_artifacts, boot_artifact) - if not artifact: - raise ArtifactNotFound() - path = artifact ['link']['path'] - etag = artifact['link']['etag'] - obj = S3Object(path, etag) - _ = obj.object_header - except ArtifactNotFound as err: - raise BootSetWarning( - f"{image_metadata.manifest_s3_url.url} doesn't contain a {boot_artifact}") from err - except Exception as err: - raise BootSetWarning( - f"Unable to check {boot_artifact} in {image_metadata.manifest_s3_url.url}. " - f"Warning: {exc_type_msg(err)}") diff --git a/src/bos/server/controllers/v2/boot_set/__init__.py b/src/bos/server/controllers/v2/boot_set/__init__.py new file mode 100644 index 00000000..95e78017 --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/__init__.py @@ -0,0 +1,29 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from .defs import DEFAULT_ARCH, HARDWARE_SPECIFIER_FIELDS, BootSetStatus +from .exceptions import BootSetArchMismatch, BootSetError, BootSetWarning, \ + CannotValidateBootSetArch, NonImsImage +from .sanitize import validate_sanitize_boot_sets +from .validate import validate_boot_sets diff --git a/src/bos/server/controllers/v2/boot_set/artifacts.py b/src/bos/server/controllers/v2/boot_set/artifacts.py new file mode 100644 index 00000000..b21a411d --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/artifacts.py @@ -0,0 +1,67 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from bos.common.utils import exc_type_msg +from bos.operators.utils.boot_image_metadata.factory import BootImageMetaDataFactory +from bos.operators.utils.clients.s3 import S3Object, ArtifactNotFound + +from .exceptions import BootSetError, BootSetWarning + + +def validate_boot_artifacts(bs: dict): + # Verify that the boot artifacts exist + try: + image_metadata = BootImageMetaDataFactory(bs)() + except Exception as err: + raise BootSetError(f"Can't find boot artifacts. Error: {exc_type_msg(err)}") from err + + # Check boot artifacts' S3 headers + for boot_artifact in ["kernel"]: + try: + artifact = getattr(image_metadata.boot_artifacts, boot_artifact) + path = artifact ['link']['path'] + etag = artifact['link']['etag'] + obj = S3Object(path, etag) + _ = obj.object_header + except Exception as err: + raise BootSetError(f"Can't find {boot_artifact} in " + f"{image_metadata.manifest_s3_url.url}. " + f"Error: {exc_type_msg(err)}") from err + + for boot_artifact in ["initrd", "boot_parameters"]: + try: + artifact = getattr(image_metadata.boot_artifacts, boot_artifact) + if not artifact: + raise ArtifactNotFound() + path = artifact ['link']['path'] + etag = artifact['link']['etag'] + obj = S3Object(path, etag) + _ = obj.object_header + except ArtifactNotFound as err: + raise BootSetWarning( + f"{image_metadata.manifest_s3_url.url} doesn't contain a {boot_artifact}") from err + except Exception as err: + raise BootSetWarning( + f"Unable to check {boot_artifact} in {image_metadata.manifest_s3_url.url}. " + f"Warning: {exc_type_msg(err)}") from err diff --git a/src/bos/server/controllers/v2/boot_set/defs.py b/src/bos/server/controllers/v2/boot_set/defs.py new file mode 100644 index 00000000..a9478016 --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/defs.py @@ -0,0 +1,41 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from enum import Enum +import logging + +LOGGER = logging.getLogger('bos.server.controllers.v2.boot_set') + +class BootSetStatus(Enum): + """ + In ascending order of error severity + """ + SUCCESS = 0 + WARNING = 1 + ERROR = 2 + +# Valid boot sets are required to have at least one of these fields +HARDWARE_SPECIFIER_FIELDS = ( "node_list", "node_roles_groups", "node_groups" ) + +DEFAULT_ARCH = "X86" diff --git a/src/bos/server/controllers/v2/boot_set/exceptions.py b/src/bos/server/controllers/v2/boot_set/exceptions.py new file mode 100644 index 00000000..1722f536 --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/exceptions.py @@ -0,0 +1,51 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +class BootSetError(Exception): + """ + Generic error class for fatal problems found during boot set validation + """ + + +class BootSetWarning(Exception): + """ + Generic error class for non-fatal problems found during boot set validation + """ + + +class NonImsImage(BootSetWarning): + """ + Raised to indicate the boot set boot image is not from IMS + """ + + +class BootSetArchMismatch(BootSetError): + def __init__(self, bs_arch: str, expected_ims_arch: str, actual_ims_arch: str): + super().__init__(f"Boot set arch '{bs_arch}' means IMS image arch should be " + f"'{expected_ims_arch}', but actual IMS image arch is '{actual_ims_arch}'") + + +class CannotValidateBootSetArch(BootSetWarning): + def __init__(self, msg: str): + super().__init__(f"Can't validate boot image arch: {msg}") diff --git a/src/bos/server/controllers/v2/boot_set/ims.py b/src/bos/server/controllers/v2/boot_set/ims.py new file mode 100644 index 00000000..0df4a165 --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/ims.py @@ -0,0 +1,123 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from bos.common.utils import exc_type_msg, requests_retry_session +from bos.operators.utils.clients.ims import get_arch_from_image_data, get_image, \ + get_ims_id_from_s3_url, ImageNotFound +from bos.operators.utils.clients.s3 import S3Url +from bos.server.controllers.v2.options import OptionsData + +from .defs import DEFAULT_ARCH +from .exceptions import BootSetArchMismatch, BootSetError, BootSetWarning, \ + CannotValidateBootSetArch, NonImsImage + +# Mapping from BOS boot set arch values to expected IMS image arch values +# Omits BOS Other value, since there is no corresponding IMS image arch value +EXPECTED_IMS_ARCH = { + "ARM": "aarch64", + "Unknown": "x86_64", + "X86": "x86_64" +} + + +def validate_ims_boot_image(bs: dict, options_data: OptionsData) -> None: + """ + If the boot set architecture is not set to Other, check that the IMS image + architecture matches the boot set architecture (treating a boot set architecture + of Unknown as X86) + + Otherwise, at least validate whether the boot image is in IMS, if we expect it to be. + """ + try: + bs_path = bs["path"] + except KeyError as err: + raise BootSetError("Missing required 'path' field") from err + + bs_arch = bs.get("arch", DEFAULT_ARCH) + + ims_id = get_ims_image_id(bs_path) + + # If IMS being inaccessible is not a fatal error, then reduce the number + # of retries we make, to prevent a lengthy delay + num_retries = 8 if options_data.ims_errors_fatal else 4 + + try: + image_data = get_ims_image_data(ims_id, num_retries) + except ImageNotFound as err: + if options_data.ims_images_must_exist: + raise BootSetError(str(err)) from err + raise BootSetWarning(str(err)) from err + except Exception as err: + if options_data.ims_errors_fatal: + raise BootSetError(exc_type_msg(err)) from err + if bs_arch != 'Other': + # This means that this error is preventing us from validating the + # boot set architecture + raise CannotValidateBootSetArch(str(err)) from err + # We weren't going to be validating the architecture, since it is Other, + # but we should still log this as a warning + raise BootSetWarning(str(err)) from err + + if bs_arch == 'Other': + raise CannotValidateBootSetArch("Boot set arch set to 'Other'") + + try: + ims_image_arch = get_arch_from_image_data(image_data) + except Exception as err: + # This most likely indicates that the IMS image data we got wasn't even a dict + if options_data.ims_errors_fatal: + raise BootSetError(exc_type_msg(err)) from err + raise BootSetWarning(str(err)) from err + + if EXPECTED_IMS_ARCH[bs_arch] != ims_image_arch: + raise BootSetArchMismatch(bs_arch=bs_arch, expected_ims_arch=EXPECTED_IMS_ARCH[bs_arch], + actual_ims_arch=ims_image_arch) + + +def get_ims_image_id(path: str) -> str: + """ + If the image is an IMS image, return its ID. + Raise NonImsImage otherwise, + Note that this does not actually check IMS to see if the ID + exists. + """ + s3_url = S3Url(path) + ims_id = get_ims_id_from_s3_url(s3_url) + if ims_id: + return ims_id + raise NonImsImage(f"Boot artifact S3 URL '{s3_url.url}' doesn't follow convention " + "for IMS images") + + +def get_ims_image_data(ims_id: str, num_retries: int|None=None) -> dict: + """ + Query IMS to get the image data and return it, + or raise an exception. + """ + kwargs = { "image_id": ims_id } + if num_retries is not None: + # A pylint bug generates a false positive error for this call + # https://github.com/pylint-dev/pylint/issues/2271 + kwargs['session'] = requests_retry_session(retries=4) # pylint: disable=redundant-keyword-arg + return get_image(**kwargs) diff --git a/src/bos/server/controllers/v2/boot_set/sanitize.py b/src/bos/server/controllers/v2/boot_set/sanitize.py new file mode 100644 index 00000000..36301ba2 --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/sanitize.py @@ -0,0 +1,117 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from bos.common.utils import exc_type_msg +from bos.server.controllers.v2.options import OptionsData +from bos.server.utils import canonize_xname + +from .artifacts import validate_boot_artifacts +from .defs import DEFAULT_ARCH, LOGGER +from .exceptions import BootSetError, BootSetWarning +from .ims import validate_ims_boot_image +from .validate import check_node_list_for_nids, verify_nonempty_hw_specifier_field + + +def validate_sanitize_boot_sets(template_data: dict, options_data: OptionsData|None=None) -> None: + """ + Calls validate_sanitize_boot_set on every boot set in the template. + Raises an exception if there are problems. + """ + # The boot_sets field is required. + try: + boot_sets = template_data["boot_sets"] + except KeyError as exc: + raise BootSetError("Missing required 'boot_sets' field") from exc + + # The boot_sets field must map to a dict + if not isinstance(boot_sets, dict): + raise BootSetError("'boot_sets' field has invalid type") + + # The boot_sets field must be non-empty + if not boot_sets: + raise BootSetError("Session templates must contain at least one boot set") + + if options_data is None: + options_data = OptionsData() + + # Finally, call validate_sanitize_boot_set on each boot set + for bs_name, bs in boot_sets.items(): + validate_sanitize_boot_set(bs_name, bs, options_data=options_data) + + +def validate_sanitize_boot_set(bs_name: str, bs_data: dict, options_data: OptionsData) -> None: + """ + Called when creating/updating a BOS session template. + Validates the boot set, and sanitizes it (editing it in place). + Since this request has come in through the API, we assume that schema-level validation has + already happened. + Raises BootSetError on error. + """ + if "name" not in bs_data: + # Set the field here -- this allows the name to be validated + # per the schema later + bs_data["name"] = bs_name + elif bs_data["name"] != bs_name: + # All keys in the boot_sets mapping must match the 'name' fields in the + # boot sets to which they map (if they contain a 'name' field). + raise BootSetError(f"boot_sets key ({bs_name}) does not match 'name' " + f"field of corresponding boot set ({bs_data['name']})") + + # Set the 'arch' field to the default value, if it is not present + if "arch" not in bs_data: + bs_data["arch"] = DEFAULT_ARCH + + # Check the boot artifacts + try: + validate_boot_artifacts(bs_data) + except (BootSetError, BootSetWarning) as err: + LOGGER.warning(str(err)) + + # Validate the boot set IMS image + try: + validate_ims_boot_image(bs_data, options_data) + except BootSetWarning as err: + LOGGER.warning("Boot set '%s': %s", bs_name, err) + LOGGER.warning('Boot set contents: %s', bs_data) + except Exception as err: + raise BootSetError(exc_type_msg(err)) from err + + # Validate that the boot set has at least one non-empty HARDWARE_SPECIFIER_FIELDS + try: + verify_nonempty_hw_specifier_field(bs_data) + except BootSetError as err: + raise BootSetError(f"Boot set {bs_name}: {err}") from err + + # Last things to do are validate/sanitize the node_list field, if it is present + if "node_list" not in bs_data: + return + + try: + check_node_list_for_nids(bs_data, options_data) + except BootSetWarning as err: + LOGGER.warning("Boot set %s: %s", bs_name, err) + except BootSetError as err: + raise BootSetError(f"Boot set {bs_name}: {err}") from err + + bs_data["node_list"] = [ canonize_xname(node) for node in bs_data["node_list"] ] diff --git a/src/bos/server/controllers/v2/boot_set/validate.py b/src/bos/server/controllers/v2/boot_set/validate.py new file mode 100644 index 00000000..415886b3 --- /dev/null +++ b/src/bos/server/controllers/v2/boot_set/validate.py @@ -0,0 +1,150 @@ +# +# MIT License +# +# (C) Copyright 2021-2022, 2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +from functools import partial +from bos.common.utils import exc_type_msg +from bos.server.controllers.v2.options import OptionsData + +from .artifacts import validate_boot_artifacts +from .defs import HARDWARE_SPECIFIER_FIELDS, LOGGER, BootSetStatus +from .exceptions import BootSetError, BootSetWarning +from .ims import validate_ims_boot_image + + +def validate_boot_sets(session_template: dict, + operation: str, + template_name: str, + options_data: OptionsData|None=None) -> tuple[str, BootSetStatus]: + """ + Validates the boot sets listed in a session template. + This is called when creating a session or when using the sessiontemplatesvalid endpoint + + It ensures that there are boot sets. + It checks that each boot set specifies nodes via at least one of the specifier fields. + Ensures that the boot artifacts exist. + + Inputs: + session_template (dict): Session template data + operation (str): Requested operation + template_name (str): The name of the session template; Note, during Session template + creation, the name in the session template data does not have + to match the name used to create the session template. + options_data (OptionsData): BOS options, or None (in which case they will be loaded from BOS) + Returns: + Returns an status code and a message + See BootSetStatus definition for details on the status codes + """ + # Verify boot sets exist. + if not session_template.get('boot_sets', None): + msg = f"Session template '{template_name}' requires at least 1 boot set." + return BootSetStatus.ERROR, msg + + if options_data is None: + options_data = OptionsData() + + warning_msgs = [] + for bs_name, bs in session_template['boot_sets'].items(): + bs_msg = partial(_bs_msg, template_name=template_name, bs_name=bs_name) + try: + bs_warning_msgs = validate_boot_set(bs=bs, operation=operation, + options_data=options_data) + except BootSetError as err: + msg = bs_msg(str(err)) + LOGGER.error(msg) + return BootSetStatus.ERROR, msg + except Exception as err: + LOGGER.error( + bs_msg(f"Unexpected exception in _validate_boot_set: {exc_type_msg(err)}")) + raise + for msg in map(bs_msg, bs_warning_msgs): + LOGGER.warning(msg) + warning_msgs.append(msg) + + if warning_msgs: + return BootSetStatus.WARNING, "; ".join(warning_msgs) + + return BootSetStatus.SUCCESS, "Valid" + + +def _bs_msg(msg: str, template_name: str, bs_name: str) -> str: + """ + Shortcut for creating validation error/warning messages for a specific bootset + """ + return f"Session template: '{template_name}' boot set: '{bs_name}': {msg}" + + +def validate_boot_set(bs: dict, operation: str, options_data: OptionsData) -> list[str]: + """ + Helper function for validate_boot_sets that performs validation on a single boot set. + Raises BootSetError if fatal errors found. + Returns a list of warning messages (if any) + """ + warning_msgs = [] + + verify_nonempty_hw_specifier_field(bs) + + try: + check_node_list_for_nids(bs, options_data) + except BootSetWarning as err: + warning_msgs.append(str(err)) + + if operation in ['boot', 'reboot']: + try: + validate_boot_artifacts(bs) + except BootSetWarning as err: + warning_msgs.append(str(err)) + + try: + validate_ims_boot_image(bs, options_data) + except BootSetWarning as err: + warning_msgs.append(str(err)) + + return warning_msgs + + +def verify_nonempty_hw_specifier_field(bs: dict) -> None: + """ + Raises an exception if there are no non-empty hardware specifier fields. + """ + # Validate that the boot set has at least one of the HARDWARE_SPECIFIER_FIELDS + if not any(field_name in bs for field_name in HARDWARE_SPECIFIER_FIELDS): + raise BootSetError(f"No hardware specifier fields ({HARDWARE_SPECIFIER_FIELDS})") + + # Validate that at least one of the HARDWARE_SPECIFIER_FIELDS is non-empty + if not any(field_name in bs and bs[field_name] + for field_name in HARDWARE_SPECIFIER_FIELDS): + raise BootSetError(f"No non-empty hardware specifier fields ({HARDWARE_SPECIFIER_FIELDS})") + + +def check_node_list_for_nids(bs: dict, options_data: OptionsData) -> None: + """ + If the node list contains no NIDs, return. + Otherwise, raise BootSetError or BootSetWarning, depending on the value of the + reject_nids option + """ + if "node_list" not in bs: + return + if any(node[:3] == "nid" for node in bs["node_list"]): + msg = "Has NID in 'node_list'" + raise BootSetError(msg) if options_data.reject_nids else BootSetWarning(msg) diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index 7c3d5406..9a5aaa0e 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -35,13 +35,14 @@ from bos.common.utils import exc_type_msg, get_current_time, get_current_timestamp, load_timestamp from bos.common.values import Phase, Status from bos.server import redis_db_utils as dbutils +from bos.server.controllers.v2.boot_set import BootSetStatus, validate_boot_sets from bos.server.controllers.v2.components import get_v2_components_data from bos.server.controllers.v2.options import OptionsData from bos.server.controllers.v2.sessiontemplates import get_v2_sessiontemplate from bos.server.models.v2_session import V2Session as Session # noqa: E501 from bos.server.models.v2_session_create import V2SessionCreate as SessionCreate # noqa: E501 from bos.server.utils import get_request_json, ParsingException -from .boot_set import validate_boot_sets, BOOT_SET_ERROR + LOGGER = logging.getLogger('bos.server.controllers.v2.session') DB = dbutils.get_wrapper(db='sessions') @@ -104,7 +105,7 @@ def post_v2_session(): # noqa: E501 # Validate health/validity of the sessiontemplate before creating a session error_code, msg = validate_boot_sets(session_template, session_create.operation, template_name, options_data=options_data) - if error_code >= BOOT_SET_ERROR: + if error_code >= BootSetStatus.ERROR: LOGGER.error("Session template fails check: %s", msg) return msg, 400