diff --git a/CHANGELOG.md b/CHANGELOG.md index 02b1c9f8..b7fb385e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update API spec to reflect that no v1-format session template will exist inside BOS, because the v1 session template creation endpoint will strip v1-specific fields and create a v2-format session template, and even the v1 session template template endpoint will return a v2-compatible example template. +- Update BOS migration code to properly convert v1 session templates to v2, both from old Etcd database and within + current redis DB. ## [2.6.3] - 08-22-2023 ### Changed diff --git a/src/bos/server/controllers/v1/sessiontemplate.py b/src/bos/server/controllers/v1/sessiontemplate.py index a9b8ed85..28858a71 100644 --- a/src/bos/server/controllers/v1/sessiontemplate.py +++ b/src/bos/server/controllers/v1/sessiontemplate.py @@ -56,6 +56,8 @@ "configuration": "desired-cfs-config"}, "enable_cfs": True} +V1_SPECIFIC_ST_FIELDS = [ "cfs_url", "cfs_branch", "partition" ] +V1_SPECIFIC_BOOTSET_FIELDS = [ "network", "boot_ordinal", "shutdown_ordinal" ] def sanitize_xnames(st_json): """ @@ -76,6 +78,33 @@ def sanitize_xnames(st_json): st_json['boot_sets'][boot_set]['node_list'] = clean_nl return st_json +def strip_v1_only_fields(template_data): + """ + Edits in-place the template data, removing any fields which are specific to BOS v1. + Returns True if any changes were made. + Returns False if nothing was removed. + """ + changes_made=False + # Strip out the v1-specific fields from the dictionary + for v1_field_name in V1_SPECIFIC_ST_FIELDS: + try: + del template_data[v1_field_name] + changes_made=True + except KeyError: + pass + + # Do the same for each boot set + # Oddly, boot_sets is not a required field, so only do this if it is present + if "boot_sets" in template_data: + for bs in template_data["boot_sets"].values(): + for v1_bs_field_name in V1_SPECIFIC_BOOTSET_FIELDS: + try: + del bs[v1_bs_field_name] + changes_made=True + except KeyError: + pass + + return changes_made @no_v1_multi_tenancy_support @dbutils.redis_error_handler @@ -111,24 +140,7 @@ def create_v1_sessiontemplate(): # noqa: E501 status=400, title="The session template could not be created.", detail=str(err)) - # Strip out the v1-specific fields from the dictionary - v1_specific_st_fields = [ "cfs_url", "cfs_branch", "partition" ] - for v1_field_name in v1_specific_st_fields: - try: - del template_data[v1_field_name] - except KeyError: - pass - - # Do the same for each boot set - # Oddly, boot_sets is not a required field, so only do this if it is present - if "boot_sets" in template_data: - v1_specific_bootset_fields = [ "network", "boot_ordinal", "shutdown_ordinal" ] - for bs in template_data["boot_sets"].values(): - for v1_bs_field_name in v1_specific_bootset_fields: - try: - del bs[v1_bs_field_name] - except KeyError: - pass + strip_v1_only_fields(template_data) # BOS v2 doesn't want the session template name inside the dictionary itself # name is a required v1 field, though, so we can safely pop it here diff --git a/src/bos/server/migrations.py b/src/bos/server/migrations.py index 63c37488..c4f65c19 100644 --- a/src/bos/server/migrations.py +++ b/src/bos/server/migrations.py @@ -26,6 +26,7 @@ import logging import os +from bos.server.controllers.v1.sessiontemplate import strip_v1_only_fields from bos.server.dbclient import BosEtcdClient from bos.common.utils import requests_retry_session from bos.common.tenant_utils import get_tenant_aware_key @@ -62,9 +63,11 @@ def pod_ip(): pods = v1.list_namespaced_pod("services", label_selector=f"app.kubernetes.io/name=cray-bos,app.kubernetes.io/version={version}") # Get the pod's IP address - if pods: - pod_ip = pods.items[0].status.pod_ip - return pod_ip + if pods and pods.items: + return pods.items[0].status.pod_ip + msg = "Could not determine BOS pod IP address. Aborting." + LOGGER.error(msg) + raise ValueError(msg) def convert_v1_to_v2(v1_st): @@ -84,8 +87,7 @@ def convert_v1_to_v2(v1_st): exception. """ session_template_keys = ['name', 'description', - 'enable_cfs', 'cfs', 'partition', - 'boot_sets', 'links'] + 'enable_cfs', 'cfs', 'boot_sets', 'links'] boot_set_keys = ['name', 'path', 'type', 'etag', 'kernel_parameters', 'node_list', 'node_roles_groups', 'node_groups', 'rootfs_provider', 'rootfs_provider_passthrough'] @@ -97,7 +99,7 @@ def convert_v1_to_v2(v1_st): raise MissingName() for k, v in v1_st.items(): if k in session_template_keys: - if k != "boot_sets" and k != "name": + if k != "boot_sets" and k != "name" and k!= "links": v2_st[k] = v else: LOGGER.warning("Discarding attribute: '{}' from session template: '{}'".format(k, v1_st['name'])) @@ -114,7 +116,7 @@ def convert_v1_to_v2(v1_st): return v2_st, name -def migrate_v1_to_v2_session_templates(): +def migrate_v1_etcd_to_v2_redis_session_templates(): """ Read the session templates out of the V1 etcd key/value store and write them into the v2 Redis database. @@ -153,20 +155,23 @@ def migrate_v1_to_v2_session_templates(): "to error: {}".format(v1_st['name'], response.reason)) LOGGER.error("Error specifics: {}".format(response.text)) - - v1_st["name"] = v1_st["name"] + "_v1_deprecated" - response = session.post("{}".format(st_v1_endpoint), json=v1_st) - if not response.ok: - LOGGER.error("Session template: '{}' was not migrated for v1 due " - "to error: {}".format(v1_st['name'], - response.reason)) - LOGGER.error("Error specifics: {}".format(response.text)) else: LOGGER.error("Session template: '{}' was not migrated due " "to error: {}".format(v1_st['name'], response.reason)) LOGGER.error("Error specifics: {}".format(response.text)) +# Convert existing v1 session templates to v2 format +def convert_v1_to_v2_session_templates(): + db=dbutils.get_wrapper(db='session_templates') + response = db.get_keys() + for st_key in response: + data = db.get(st_key) + if strip_v1_only_fields(data): + name = data.get("name") + LOGGER.info(f"Converting {name} to BOS v2") + db.put(st_key, data) + # Multi-tenancy key migrations def migrate_database(db): @@ -188,8 +193,9 @@ def migrate_to_tenant_aware_keys(): def perform_migrations(): - migrate_v1_to_v2_session_templates() + migrate_v1_etcd_to_v2_redis_session_templates() migrate_to_tenant_aware_keys() + convert_v1_to_v2_session_templates() if __name__ == "__main__":