Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CASMCMS-8799: Create v2-compatible session templates with v1 endpoint #212

Merged
merged 2 commits into from
Sep 12, 2023
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Update API spec to reflect that no v1-format session template will exist inside BOS, because the
v1 session template creation endpoint will strip v1-specific fields and create a v2-format session template,
and even the v1 session template template endpoint will return a v2-compatible example template.
- Update BOS migration code to properly convert v1 session templates to v2, both from old Etcd database and within
current redis DB.

## [2.6.3] - 08-22-2023
### Changed
48 changes: 30 additions & 18 deletions src/bos/server/controllers/v1/sessiontemplate.py
Original file line number Diff line number Diff line change
@@ -56,6 +56,8 @@
"configuration": "desired-cfs-config"},
"enable_cfs": True}

V1_SPECIFIC_ST_FIELDS = [ "cfs_url", "cfs_branch", "partition" ]
V1_SPECIFIC_BOOTSET_FIELDS = [ "network", "boot_ordinal", "shutdown_ordinal" ]

def sanitize_xnames(st_json):
"""
@@ -76,6 +78,33 @@ def sanitize_xnames(st_json):
st_json['boot_sets'][boot_set]['node_list'] = clean_nl
return st_json

def strip_v1_only_fields(template_data):
"""
Edits in-place the template data, removing any fields which are specific to BOS v1.
Returns True if any changes were made.
Returns False if nothing was removed.
"""
changes_made=False
# Strip out the v1-specific fields from the dictionary
for v1_field_name in V1_SPECIFIC_ST_FIELDS:
try:
del template_data[v1_field_name]
changes_made=True
except KeyError:
pass

# Do the same for each boot set
# Oddly, boot_sets is not a required field, so only do this if it is present
if "boot_sets" in template_data:
for bs in template_data["boot_sets"].values():
for v1_bs_field_name in V1_SPECIFIC_BOOTSET_FIELDS:
try:
del bs[v1_bs_field_name]
changes_made=True
except KeyError:
pass

return changes_made

@no_v1_multi_tenancy_support
@dbutils.redis_error_handler
@@ -111,24 +140,7 @@ def create_v1_sessiontemplate(): # noqa: E501
status=400, title="The session template could not be created.",
detail=str(err))

# Strip out the v1-specific fields from the dictionary
v1_specific_st_fields = [ "cfs_url", "cfs_branch", "partition" ]
for v1_field_name in v1_specific_st_fields:
try:
del template_data[v1_field_name]
except KeyError:
pass

# Do the same for each boot set
# Oddly, boot_sets is not a required field, so only do this if it is present
if "boot_sets" in template_data:
v1_specific_bootset_fields = [ "network", "boot_ordinal", "shutdown_ordinal" ]
for bs in template_data["boot_sets"].values():
for v1_bs_field_name in v1_specific_bootset_fields:
try:
del bs[v1_bs_field_name]
except KeyError:
pass
strip_v1_only_fields(template_data)

# BOS v2 doesn't want the session template name inside the dictionary itself
# name is a required v1 field, though, so we can safely pop it here
38 changes: 22 additions & 16 deletions src/bos/server/migrations.py
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@
import logging
import os

from bos.server.controllers.v1.sessiontemplate import strip_v1_only_fields
from bos.server.dbclient import BosEtcdClient
from bos.common.utils import requests_retry_session
from bos.common.tenant_utils import get_tenant_aware_key
@@ -62,9 +63,11 @@ def pod_ip():
pods = v1.list_namespaced_pod("services",
label_selector=f"app.kubernetes.io/name=cray-bos,app.kubernetes.io/version={version}")
# Get the pod's IP address
if pods:
pod_ip = pods.items[0].status.pod_ip
return pod_ip
if pods and pods.items:
return pods.items[0].status.pod_ip
msg = "Could not determine BOS pod IP address. Aborting."
LOGGER.error(msg)
raise ValueError(msg)


def convert_v1_to_v2(v1_st):
@@ -84,8 +87,7 @@ def convert_v1_to_v2(v1_st):
exception.
"""
session_template_keys = ['name', 'description',
'enable_cfs', 'cfs', 'partition',
'boot_sets', 'links']
'enable_cfs', 'cfs', 'boot_sets', 'links']
boot_set_keys = ['name', 'path', 'type', 'etag', 'kernel_parameters',
'node_list', 'node_roles_groups', 'node_groups',
'rootfs_provider', 'rootfs_provider_passthrough']
@@ -97,7 +99,7 @@ def convert_v1_to_v2(v1_st):
raise MissingName()
for k, v in v1_st.items():
if k in session_template_keys:
if k != "boot_sets" and k != "name":
if k != "boot_sets" and k != "name" and k!= "links":
v2_st[k] = v
else:
LOGGER.warning("Discarding attribute: '{}' from session template: '{}'".format(k, v1_st['name']))
@@ -114,7 +116,7 @@ def convert_v1_to_v2(v1_st):
return v2_st, name


def migrate_v1_to_v2_session_templates():
def migrate_v1_etcd_to_v2_redis_session_templates():
"""
Read the session templates out of the V1 etcd key/value store and
write them into the v2 Redis database.
@@ -153,20 +155,23 @@ def migrate_v1_to_v2_session_templates():
"to error: {}".format(v1_st['name'],
response.reason))
LOGGER.error("Error specifics: {}".format(response.text))

v1_st["name"] = v1_st["name"] + "_v1_deprecated"
response = session.post("{}".format(st_v1_endpoint), json=v1_st)
if not response.ok:
LOGGER.error("Session template: '{}' was not migrated for v1 due "
"to error: {}".format(v1_st['name'],
response.reason))
LOGGER.error("Error specifics: {}".format(response.text))
else:
LOGGER.error("Session template: '{}' was not migrated due "
"to error: {}".format(v1_st['name'],
response.reason))
LOGGER.error("Error specifics: {}".format(response.text))

# Convert existing v1 session templates to v2 format
def convert_v1_to_v2_session_templates():
db=dbutils.get_wrapper(db='session_templates')
response = db.get_keys()
for st_key in response:
data = db.get(st_key)
if strip_v1_only_fields(data):
name = data.get("name")
LOGGER.info(f"Converting {name} to BOS v2")
db.put(st_key, data)

# Multi-tenancy key migrations

def migrate_database(db):
@@ -188,8 +193,9 @@ def migrate_to_tenant_aware_keys():


def perform_migrations():
migrate_v1_to_v2_session_templates()
migrate_v1_etcd_to_v2_redis_session_templates()
migrate_to_tenant_aware_keys()
convert_v1_to_v2_session_templates()


if __name__ == "__main__":