diff --git a/CHANGELOG.md b/CHANGELOG.md index c5b1ecbc..b7fb385e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Removed non-v2 fields from v1 session template template - Provide more useful example values in v1 and v2 session template templates +- Modify v1 session template create function to strip away v1-specific fields and create a v2-compatible + session template. +- Update API spec to reflect that no v1-format session template will exist inside BOS, because the + v1 session template creation endpoint will strip v1-specific fields and create a v2-format session template, + and even the v1 session template template endpoint will return a v2-compatible example template. +- Update BOS migration code to properly convert v1 session templates to v2, both from old Etcd database and within + current redis DB. ## [2.6.3] - 08-22-2023 ### Changed diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 64952cc5..9af42365 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1197,6 +1197,11 @@ components: $ref: '#/components/schemas/BootSetRootfsProviderPassthrough' additionalProperties: false required: [path, type] + V2SessionTemplateArray: + description: An array of Session Templates. + type: array + items: + $ref: '#/components/schemas/V2SessionTemplate' V2Session: description: | A Session object @@ -1595,14 +1600,7 @@ components: minimum: 0 maximum: 1048576 additionalProperties: true - # Schemas that combine objects of different versions - SessionTemplateArray: - description: An array of Session Templates. - type: array - items: - anyOf: - - $ref: '#/components/schemas/V1SessionTemplate' - - $ref: '#/components/schemas/V2SessionTemplate' + requestBodies: V2sessionCreateRequest: description: The information to create a Session @@ -1703,12 +1701,6 @@ components: application/json: schema: $ref: '#/components/schemas/V1SessionStatus' - V1SessionTemplateDetails: - description: Session Template details - content: - application/json: - schema: - $ref: '#/components/schemas/V1SessionTemplate' # V2 V2SessionTemplateDetails: description: Session Template details @@ -1716,6 +1708,12 @@ components: application/json: schema: $ref: '#/components/schemas/V2SessionTemplate' + V2SessionTemplateDetailsArray: + description: Session Template details array + content: + application/json: + schema: + $ref: '#/components/schemas/V2SessionTemplateArray' V2SessionTemplateValidation: description: Session Template validity details content: @@ -1764,21 +1762,6 @@ components: application/json: schema: $ref: '#/components/schemas/V2Options' - # Responses that may contain V1 or V2 objects - SessionTemplateDetails: - description: Session Template details - content: - application/json: - schema: - anyOf: - - $ref: '#/components/schemas/V1SessionTemplate' - - $ref: '#/components/schemas/V2SessionTemplate' - SessionTemplateDetailsArray: - description: Session Template details array - content: - application/json: - schema: - $ref: '#/components/schemas/SessionTemplateArray' # Errors AlreadyExists: description: The resource to be created already exists @@ -1978,7 +1961,10 @@ paths: - $ref: '#/components/parameters/V1TenantHeaderParam' post: summary: Create Session Template - description: Create a new Session Template. + description: | + Create a new Session Template. + + The created template will be modified if necessary to follow the BOS v2 session template format. tags: - sessiontemplate x-openapi-router-controller: bos.server.controllers.v1.sessiontemplate @@ -2005,7 +1991,7 @@ paths: operationId: get_v1_sessiontemplates responses: 200: - $ref: '#/components/responses/SessionTemplateDetailsArray' + $ref: '#/components/responses/V2SessionTemplateDetailsArray' 400: $ref: '#/components/responses/MultiTenancyNotSupported' /v1/sessiontemplate/{session_template_id}: @@ -2024,7 +2010,7 @@ paths: operationId: get_v1_sessiontemplate responses: 200: - $ref: '#/components/responses/SessionTemplateDetails' + $ref: '#/components/responses/V2SessionTemplateDetails' 400: $ref: '#/components/responses/MultiTenancyNotSupported' 404: @@ -2056,7 +2042,7 @@ paths: operationId: get_v1_sessiontemplatetemplate responses: 200: - $ref: '#/components/responses/V1SessionTemplateDetails' + $ref: '#/components/responses/V2SessionTemplateDetails' /v1/session: parameters: - $ref: '#/components/parameters/V1TenantHeaderParam' @@ -2411,7 +2397,7 @@ paths: operationId: get_v2_sessiontemplates responses: 200: - $ref: '#/components/responses/SessionTemplateDetailsArray' + $ref: '#/components/responses/V2SessionTemplateDetailsArray' /v2/sessiontemplatesvalid/{session_template_id}: parameters: - $ref: '#/components/parameters/TemplateIdPathParam' @@ -2449,7 +2435,7 @@ paths: operationId: get_v2_sessiontemplate responses: 200: - $ref: '#/components/responses/SessionTemplateDetails' + $ref: '#/components/responses/V2SessionTemplateDetails' 404: $ref: '#/components/responses/ResourceNotFound' put: diff --git a/src/bos/server/controllers/v1/sessiontemplate.py b/src/bos/server/controllers/v1/sessiontemplate.py index 2936751a..28858a71 100644 --- a/src/bos/server/controllers/v1/sessiontemplate.py +++ b/src/bos/server/controllers/v1/sessiontemplate.py @@ -30,7 +30,8 @@ from bos.common.tenant_utils import no_v1_multi_tenancy_support from bos.server import redis_db_utils as dbutils -from bos.server.models.v1_session_template import V1SessionTemplate as SessionTemplate # noqa: E501 +from bos.server.models.v1_session_template import V1SessionTemplate # noqa: E501 +from bos.server.models.v2_session_template import V2SessionTemplate # noqa: E501 from bos.server.utils import _canonize_xname from bos.common.tenant_utils import get_tenant_aware_key from ..v2.sessiontemplates import get_v2_sessiontemplate, get_v2_sessiontemplates, delete_v2_sessiontemplate @@ -55,6 +56,8 @@ "configuration": "desired-cfs-config"}, "enable_cfs": True} +V1_SPECIFIC_ST_FIELDS = [ "cfs_url", "cfs_branch", "partition" ] +V1_SPECIFIC_BOOTSET_FIELDS = [ "network", "boot_ordinal", "shutdown_ordinal" ] def sanitize_xnames(st_json): """ @@ -75,6 +78,33 @@ def sanitize_xnames(st_json): st_json['boot_sets'][boot_set]['node_list'] = clean_nl return st_json +def strip_v1_only_fields(template_data): + """ + Edits in-place the template data, removing any fields which are specific to BOS v1. + Returns True if any changes were made. + Returns False if nothing was removed. + """ + changes_made=False + # Strip out the v1-specific fields from the dictionary + for v1_field_name in V1_SPECIFIC_ST_FIELDS: + try: + del template_data[v1_field_name] + changes_made=True + except KeyError: + pass + + # Do the same for each boot set + # Oddly, boot_sets is not a required field, so only do this if it is present + if "boot_sets" in template_data: + for bs in template_data["boot_sets"].values(): + for v1_bs_field_name in V1_SPECIFIC_BOOTSET_FIELDS: + try: + del bs[v1_bs_field_name] + changes_made=True + except KeyError: + pass + + return changes_made @no_v1_multi_tenancy_support @dbutils.redis_error_handler @@ -94,7 +124,8 @@ def create_v1_sessiontemplate(): # noqa: E501 sessiontemplate = None try: - """Convert the JSON request data into a SessionTemplate object. + """Verify that we can convert the JSON request data into a + V1SessionTemplate object. Any exceptions caught here would be generated from the model (i.e. bos.server.models.session_template). Examples are an exception for a session template missing the required name @@ -102,23 +133,43 @@ def create_v1_sessiontemplate(): # noqa: E501 confirm to Kubernetes naming convention. In this case return 400 with a description of the specific error. """ - sessiontemplate = SessionTemplate.from_dict(connexion.request.get_json()) + template_data = connexion.request.get_json() + V1SessionTemplate.from_dict(template_data) except Exception as err: return connexion.problem( status=400, title="The session template could not be created.", detail=str(err)) - """For now overwrite any existing template by name w/o warning. - Later this can be changed when we support patching operations. - This could also be changed to result in an HTTP 409 Conflict. TBD. - """ - LOGGER.debug("create_v1_sessiontemplate name: %s", sessiontemplate.name) - st_json = connexion.request.get_json() + strip_v1_only_fields(template_data) + + # BOS v2 doesn't want the session template name inside the dictionary itself + # name is a required v1 field, though, so we can safely pop it here + session_template_id = template_data.pop("name") + + # Now basically follow the same process as when creating a V2 session template (except in the end, + # if successful, we will return 201 status and the name of the template, to match the v1 API spec) + try: + """Verify that we can convert the JSON request data into a + V2SessionTemplate object. + Any exceptions caught here would be generated from the model + (i.e. bos.server.models.session_template). + An example is an exception for a session template name that + does not conform to Kubernetes naming convention. + In this case return 400 with a description of the specific error. + """ + V2SessionTemplate.from_dict(template_data) + except Exception as err: + return connexion.problem( + status=400, title="The session template could not be created as a v2 template.", + detail=str(err)) + + template_data = sanitize_xnames(template_data) + template_data['name'] = session_template_id # Tenants are not used in v1, but v1 and v2 share template storage - st_json["tenant"] = "" - template_key = get_tenant_aware_key(sessiontemplate.name, "") - DB.put(template_key, st_json) - return sessiontemplate.name, 201 + template_data['tenant'] = "" + template_key = get_tenant_aware_key(session_template_id, "") + DB.put(template_key, template_data) + return session_template_id, 201 @no_v1_multi_tenancy_support diff --git a/src/bos/server/migrations.py b/src/bos/server/migrations.py index 63c37488..c4f65c19 100644 --- a/src/bos/server/migrations.py +++ b/src/bos/server/migrations.py @@ -26,6 +26,7 @@ import logging import os +from bos.server.controllers.v1.sessiontemplate import strip_v1_only_fields from bos.server.dbclient import BosEtcdClient from bos.common.utils import requests_retry_session from bos.common.tenant_utils import get_tenant_aware_key @@ -62,9 +63,11 @@ def pod_ip(): pods = v1.list_namespaced_pod("services", label_selector=f"app.kubernetes.io/name=cray-bos,app.kubernetes.io/version={version}") # Get the pod's IP address - if pods: - pod_ip = pods.items[0].status.pod_ip - return pod_ip + if pods and pods.items: + return pods.items[0].status.pod_ip + msg = "Could not determine BOS pod IP address. Aborting." + LOGGER.error(msg) + raise ValueError(msg) def convert_v1_to_v2(v1_st): @@ -84,8 +87,7 @@ def convert_v1_to_v2(v1_st): exception. """ session_template_keys = ['name', 'description', - 'enable_cfs', 'cfs', 'partition', - 'boot_sets', 'links'] + 'enable_cfs', 'cfs', 'boot_sets', 'links'] boot_set_keys = ['name', 'path', 'type', 'etag', 'kernel_parameters', 'node_list', 'node_roles_groups', 'node_groups', 'rootfs_provider', 'rootfs_provider_passthrough'] @@ -97,7 +99,7 @@ def convert_v1_to_v2(v1_st): raise MissingName() for k, v in v1_st.items(): if k in session_template_keys: - if k != "boot_sets" and k != "name": + if k != "boot_sets" and k != "name" and k!= "links": v2_st[k] = v else: LOGGER.warning("Discarding attribute: '{}' from session template: '{}'".format(k, v1_st['name'])) @@ -114,7 +116,7 @@ def convert_v1_to_v2(v1_st): return v2_st, name -def migrate_v1_to_v2_session_templates(): +def migrate_v1_etcd_to_v2_redis_session_templates(): """ Read the session templates out of the V1 etcd key/value store and write them into the v2 Redis database. @@ -153,20 +155,23 @@ def migrate_v1_to_v2_session_templates(): "to error: {}".format(v1_st['name'], response.reason)) LOGGER.error("Error specifics: {}".format(response.text)) - - v1_st["name"] = v1_st["name"] + "_v1_deprecated" - response = session.post("{}".format(st_v1_endpoint), json=v1_st) - if not response.ok: - LOGGER.error("Session template: '{}' was not migrated for v1 due " - "to error: {}".format(v1_st['name'], - response.reason)) - LOGGER.error("Error specifics: {}".format(response.text)) else: LOGGER.error("Session template: '{}' was not migrated due " "to error: {}".format(v1_st['name'], response.reason)) LOGGER.error("Error specifics: {}".format(response.text)) +# Convert existing v1 session templates to v2 format +def convert_v1_to_v2_session_templates(): + db=dbutils.get_wrapper(db='session_templates') + response = db.get_keys() + for st_key in response: + data = db.get(st_key) + if strip_v1_only_fields(data): + name = data.get("name") + LOGGER.info(f"Converting {name} to BOS v2") + db.put(st_key, data) + # Multi-tenancy key migrations def migrate_database(db): @@ -188,8 +193,9 @@ def migrate_to_tenant_aware_keys(): def perform_migrations(): - migrate_v1_to_v2_session_templates() + migrate_v1_etcd_to_v2_redis_session_templates() migrate_to_tenant_aware_keys() + convert_v1_to_v2_session_templates() if __name__ == "__main__":