CASMCMS-8799: Update BOS migration code to properly migrate session t…

…emplates from v1 to v2

CHANGELOG.md

@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Update API spec to reflect that no v1-format session template will exist inside BOS, because the
   v1 session template creation endpoint will strip v1-specific fields and create a v2-format session template,
   and even the v1 session template template endpoint will return a v2-compatible example template.
+- Update BOS migration code to properly convert v1 session templates to v2, both from old Etcd database and within
+  current redis DB.
 
 ## [2.6.3] - 08-22-2023
 ### Changed

src/bos/server/controllers/v1/sessiontemplate.py

@@ -56,6 +56,8 @@
         "configuration": "desired-cfs-config"},
     "enable_cfs": True}
 
+V1_SPECIFIC_ST_FIELDS = [ "cfs_url", "cfs_branch", "partition" ]
+V1_SPECIFIC_BOOTSET_FIELDS = [ "network", "boot_ordinal", "shutdown_ordinal" ]
 
 def sanitize_xnames(st_json):
     """
@@ -76,6 +78,33 @@ def sanitize_xnames(st_json):
                 st_json['boot_sets'][boot_set]['node_list'] = clean_nl
     return st_json
 
+def strip_v1_only_fields(template_data):
+    """
+    Edits in-place the template data, removing any fields which are specific to BOS v1.
+    Returns True if any changes were made.
+    Returns False if nothing was removed.
+    """
+    changes_made=False
+    # Strip out the v1-specific fields from the dictionary
+    for v1_field_name in V1_SPECIFIC_ST_FIELDS:
+        try:
+            del template_data[v1_field_name]
+            changes_made=True
+        except KeyError:
+            pass
+
+    # Do the same for each boot set    
+    # Oddly, boot_sets is not a required field, so only do this if it is present
+    if "boot_sets" in template_data:
+        for bs in template_data["boot_sets"].values():
+            for v1_bs_field_name in V1_SPECIFIC_BOOTSET_FIELDS:
+                try:
+                    del bs[v1_bs_field_name]
+                    changes_made=True
+                except KeyError:
+                    pass
+
+    return changes_made
 
 @no_v1_multi_tenancy_support
 @dbutils.redis_error_handler
@@ -111,24 +140,7 @@ def create_v1_sessiontemplate():  # noqa: E501
             status=400, title="The session template could not be created.",
             detail=str(err))
 
-    # Strip out the v1-specific fields from the dictionary
-    v1_specific_st_fields = [ "cfs_url", "cfs_branch", "partition" ]
-    for v1_field_name in v1_specific_st_fields:
-        try:
-            del template_data[v1_field_name]
-        except KeyError:
-            pass
-
-    # Do the same for each boot set    
-    # Oddly, boot_sets is not a required field, so only do this if it is present
-    if "boot_sets" in template_data:
-        v1_specific_bootset_fields = [ "network", "boot_ordinal", "shutdown_ordinal" ]
-        for bs in template_data["boot_sets"].values():
-            for v1_bs_field_name in v1_specific_bootset_fields:
-                try:
-                    del bs[v1_bs_field_name]
-                except KeyError:
-                    pass
+    strip_v1_only_fields(template_data)
 
     # BOS v2 doesn't want the session template name inside the dictionary itself
     # name is a required v1 field, though, so we can safely pop it here

src/bos/server/migrations.py

@@ -25,7 +25,7 @@
 from kubernetes import client, config
 import logging
 import os
-
+from bos.server.controllers.v1.sessiontemplate import strip_v1_only_fields
 from bos.server.dbclient import BosEtcdClient
 from bos.common.utils import requests_retry_session
 from bos.common.tenant_utils import get_tenant_aware_key
@@ -38,6 +38,7 @@
 PROTOCOL = 'http'
 SERVICE_NAME = 'cray-bos'
 
+
 def MissingName():
     """
     The session template's name is missing
@@ -54,17 +55,19 @@ def pod_ip():
     config.load_incluster_config()
     v1 = client.CoreV1Api()
     # Find the correct version of the cray-bos pod
-    version = os.getenv('APP_VERSION')
+    version = os.getenv('APP_VERSION')    
     if not version:
         msg = "Could not determine application's version. Therefore could not contact the correct BOS pod. Aborting."
         LOGGER.error(msg)
         raise ValueError(msg)
     pods = v1.list_namespaced_pod("services",
                                   label_selector=f"app.kubernetes.io/name=cray-bos,app.kubernetes.io/version={version}")
     # Get the pod's IP address
-    if pods:
-        pod_ip = pods.items[0].status.pod_ip
-    return pod_ip
+    if pods and pods.items:
+        return pods.items[0].status.pod_ip
+    msg = "Could not determine BOS pod IP address. Aborting."
+    LOGGER.error(msg)
+    raise ValueError(msg)
 
 
 def convert_v1_to_v2(v1_st):
@@ -84,8 +87,7 @@ def convert_v1_to_v2(v1_st):
                    exception.
     """
     session_template_keys = ['name', 'description',
-                             'enable_cfs', 'cfs', 'partition',
-                             'boot_sets', 'links']
+                             'enable_cfs', 'cfs', 'boot_sets', 'links']
     boot_set_keys = ['name', 'path', 'type', 'etag', 'kernel_parameters',
                      'node_list', 'node_roles_groups', 'node_groups',
                      'rootfs_provider', 'rootfs_provider_passthrough']
@@ -97,7 +99,7 @@ def convert_v1_to_v2(v1_st):
         raise MissingName()
     for k, v in v1_st.items():
         if k in session_template_keys:
-            if k != "boot_sets" and k != "name":
+            if k != "boot_sets" and k != "name" and k!= "links":
                 v2_st[k] = v
         else:
             LOGGER.warning("Discarding attribute: '{}' from session template: '{}'".format(k, v1_st['name']))
@@ -114,7 +116,7 @@ def convert_v1_to_v2(v1_st):
     return v2_st, name
 
 
-def migrate_v1_to_v2_session_templates():
+def migrate_v1_etcd_to_v2_redis_session_templates():
     """
     Read the session templates out of the V1 etcd key/value store and
     write them into the v2 Redis database.
@@ -153,20 +155,23 @@ def migrate_v1_to_v2_session_templates():
                                  "to error: {}".format(v1_st['name'],
                                                        response.reason))
                     LOGGER.error("Error specifics: {}".format(response.text))
-
-                v1_st["name"] = v1_st["name"] + "_v1_deprecated"
-                response = session.post("{}".format(st_v1_endpoint), json=v1_st)
-                if not response.ok:
-                    LOGGER.error("Session template: '{}' was not migrated for v1 due "
-                                 "to error: {}".format(v1_st['name'],
-                                                       response.reason))
-                    LOGGER.error("Error specifics: {}".format(response.text))
             else:
                 LOGGER.error("Session template: '{}' was not migrated due "
                                  "to error: {}".format(v1_st['name'],
                                                        response.reason))
                 LOGGER.error("Error specifics: {}".format(response.text))
 
+# Convert existing v1 session templates to v2 format
+def convert_v1_to_v2_session_templates():
+    db=dbutils.get_wrapper(db='session_templates')
+    response = db.get_keys()
+    for st_key in response:
+        data = db.get(st_key)
+        if strip_v1_only_fields(data):
+            name = data.get("name")
+            LOGGER.info(f"Converting {name} to BOS v2")
+            db.put(st_key, data)
+
 # Multi-tenancy key migrations
 
 def migrate_database(db):
@@ -188,8 +193,9 @@ def migrate_to_tenant_aware_keys():
 
 
 def perform_migrations():
-    migrate_v1_to_v2_session_templates()
+    migrate_v1_etcd_to_v2_redis_session_templates()
     migrate_to_tenant_aware_keys()
+    convert_v1_to_v2_session_templates()
 
 
 if __name__ == "__main__":