From 4185a6ffdc84949ee6f4be7ecbec3906cd2bfc66 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 26 Jul 2024 13:31:22 -0400 Subject: [PATCH 1/7] List Python packages after installing them --- CHANGELOG.md | 3 +++ Dockerfile | 16 +++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8de405bc..aa0ecafd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- List Python packages after installing, for build log purposes + ## [2.22.0] ### Added - Add request timeouts to BOS reporter API calls diff --git a/Dockerfile b/Dockerfile index 345689c8..c0f33def 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,15 +68,20 @@ RUN apk add --upgrade --no-cache apk-tools busybox && \ ENV VIRTUAL_ENV=/app/venv RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN pip3 install --no-cache-dir -U pip -c constraints.txt -RUN --mount=type=secret,id=netrc,target=/root/.netrc pip3 install --no-cache-dir -r requirements.txt -RUN cd lib && pip3 install --no-cache-dir . -c ../constraints.txt +RUN pip3 install --no-cache-dir -U pip -c constraints.txt && \ + pip3 list --format freeze +RUN --mount=type=secret,id=netrc,target=/root/.netrc pip3 install --no-cache-dir -r requirements.txt && \ + pip3 list --format freeze +RUN cd lib && pip3 install --no-cache-dir . -c ../constraints.txt && \ + pip3 list --format freeze # Base testing image FROM base as testing WORKDIR /app COPY test-requirements.txt . -RUN --mount=type=secret,id=netrc,target=/root/.netrc cd /app && pip3 install --no-cache-dir -r test-requirements.txt +RUN --mount=type=secret,id=netrc,target=/root/.netrc cd /app && \ + pip3 install --no-cache-dir -r test-requirements.txt && \ + pip3 list --format freeze # Codestyle reporting FROM testing as codestyle @@ -104,7 +109,8 @@ FROM intermediate as debug ENV PYTHONPATH "/app/lib/server" WORKDIR /app RUN apk add --no-cache busybox-extras && \ - pip3 install --no-cache-dir rpdb -c constraints.txt + pip3 install --no-cache-dir rpdb -c constraints.txt && \ + pip3 list --format freeze # Application image FROM intermediate as application From 09b9d6c04243850103d5ae925834fbcc468e8c64 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Fri, 26 Jul 2024 13:32:36 -0400 Subject: [PATCH 2/7] Pin major/minor versions for Python packages, but use latest patch version --- CHANGELOG.md | 3 ++ constraints.txt.in | 82 +++++++++++++++++++++++----------------------- 2 files changed, 44 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0ecafd..d402d03e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - List Python packages after installing, for build log purposes +### Dependencies +- Pin major/minor versions for Python packages, but use latest patch version + ## [2.22.0] ### Added - Add request timeouts to BOS reporter API calls diff --git a/constraints.txt.in b/constraints.txt.in index 691bda26..2cdfa1bb 100644 --- a/constraints.txt.in +++ b/constraints.txt.in @@ -1,46 +1,46 @@ -async-timeout==4.0.3 -attrs==22.2.0 +async-timeout>=4.0.3,<4.1 +attrs>=22.2,<22.3 bos.server==0.0.1 -boto3==1.26.165 -botocore==1.29.165 -cachetools==5.3.3 +boto3>=1.26.165,<1.27 +botocore>=1.29.165,<1.30 +cachetools>=5.3.3,<5.4 certifi==2023.7.22 -charset-normalizer==3.1.0 -click==8.1.7 -clickclick==20.10.2 -connexion==2.14.2 -etcd3==0.12.0 -Flask==2.2.5 -google-auth==2.16.3 -grpcio==1.51.3 -idna==3.4 -inflection==0.5.1 -itsdangerous==2.1.2 -Jinja2==3.0.3 -jmespath==1.0.1 -jsonschema==4.17.3 +charset-normalizer>=3.1,<3.2 +click>=8.1.7,<8.2 +clickclick>=20.10.2,<20.11 +connexion>=2.14.2,<2.15 +etcd3>=0.12,<0.13 +Flask>=2.2.5,<2.3 +google-auth>=2.16.3,<2.17 +grpcio>=1.51.3,<1.52 +idna>=3.4,<3.5 +inflection>=0.5.1,<0.6 +itsdangerous>=2.1.2,<2.2 +Jinja2>=3.0.3,<3.1 +jmespath>=1.0.1,<1.1 +jsonschema>=4.17.3,<4.18 # CSM 1.6 uses Kubernetes 1.22, so use client v22.x to ensure compatability -kubernetes==22.6.0 +kubernetes>=22.6.0,<22.7 liveness==0.0.0-liveness -MarkupSafe==2.1.5 -oauthlib==3.2.2 -packaging==21.3 -protobuf==4.22.5 -pyasn1==0.4.8 -pyasn1-modules==0.2.8 -pyparsing==3.0.9 -pyrsistent==0.19.3 -python-dateutil==2.8.2 -PyYAML==6.0.1 -redis==4.5.5 -requests==2.28.2 -requests-oauthlib==1.3.1 -retrying==1.3.4 -rsa==4.9 -s3transfer==0.6.2 -six==1.16.0 +MarkupSafe>=2.1.5,<2.2 +oauthlib>=3.2.2,<3.3 +packaging>=21.3,<21.4 +protobuf>=4.22.5,<4.23 +pyasn1>=0.4.8,<0.5 +pyasn1-modules>=0.2.8,<0.3 +pyparsing>=3.0.9,<3.1 +pyrsistent>=0.19.3,<0.20 +python-dateutil>=2.8.2,<2.9 +PyYAML>=6.0.1,<6.1 +redis>=4.5.5,<4.6 +requests>=2.28.2,<2.29 +requests-oauthlib>=1.3.1,<1.4 +retrying>=1.3.4,<1.4 +rsa>=4.9,<4.10 +s3transfer>=0.6.2,<0.7 +six>=1.16,<1.17 swagger-ui-bundle==0.0.9 -tenacity==8.2.3 -urllib3==1.26.18 -websocket-client==1.5.3 -Werkzeug==2.2.3 +tenacity>=8.2.3,<8.3 +urllib3>=1.26.18,<1.27 +websocket-client>=1.5.3,<1.6 +Werkzeug>=2.2.3,<2.3 From 564e23747a79ab3b4da4b4f60ec8eab5b41d1680 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Tue, 23 Jul 2024 12:45:00 -0400 Subject: [PATCH 3/7] Remove unused BootSetNamePathParam schema from API spec --- CHANGELOG.md | 3 +++ api/openapi.yaml.in | 7 ------- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d402d03e..f2ba76f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - List Python packages after installing, for build log purposes +### Removed +- Removed unused `BootSetNamePathParam` schema from the API spec (a vestige of BOS v1) + ### Dependencies - Pin major/minor versions for Python packages, but use latest patch version diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 5be2f42d..18464202 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -1248,13 +1248,6 @@ components: $ref: '#/components/schemas/ProblemDetails' parameters: - BootSetNamePathParam: - name: boot_set_name - in: path - description: Boot Set name - required: true - schema: - type: string TemplateIdPathParam: name: session_template_id in: path From 73b1310278523008a0dbbbc184e7feb24ed8c7a7 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Tue, 23 Jul 2024 12:46:03 -0400 Subject: [PATCH 4/7] Removed inaccurate docstring from _sanitize_xnames; removed unnecessary return value --- CHANGELOG.md | 1 + src/bos/server/controllers/v2/sessiontemplates.py | 9 +++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ba76f9..207628b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - Removed unused `BootSetNamePathParam` schema from the API spec (a vestige of BOS v1) +- Removed inaccurate docstring from `_sanitize_xnames`; removed unnecessary return value ### Dependencies - Pin major/minor versions for Python packages, but use latest patch version diff --git a/src/bos/server/controllers/v2/sessiontemplates.py b/src/bos/server/controllers/v2/sessiontemplates.py index 3c20456b..0d0dbb00 100644 --- a/src/bos/server/controllers/v2/sessiontemplates.py +++ b/src/bos/server/controllers/v2/sessiontemplates.py @@ -58,13 +58,11 @@ def _sanitize_xnames(st_json): """ Sanitize xnames - Canonize the xnames - N.B. Because python passes object references by value you need to use - the return value. It will have no impact on the inputted object. Args: st_json (dict): The Session Template as a JSON object Returns: - The Session Template with all of the xnames sanitized + Nothing """ if 'boot_sets' in st_json: for boot_set in st_json['boot_sets']: @@ -72,7 +70,6 @@ def _sanitize_xnames(st_json): clean_nl = [_canonize_xname(node) for node in st_json['boot_sets'][boot_set]['node_list']] st_json['boot_sets'][boot_set]['node_list'] = clean_nl - return st_json @reject_invalid_tenant @@ -114,7 +111,7 @@ def put_v2_sessiontemplate(session_template_id): # noqa: E501 status=400, title="The session template could not be created.", detail=str(err)) - template_data = _sanitize_xnames(template_data) + _sanitize_xnames(template_data) tenant = get_tenant_from_header() template_data['name'] = session_template_id template_data['tenant'] = tenant @@ -229,7 +226,7 @@ def patch_v2_sessiontemplate(session_template_id): status=400, title="The session template could not be patched.", detail=str(err)) - template_data = _sanitize_xnames(template_data) + _sanitize_xnames(template_data) template_data['name'] = session_template_id return DB.patch(template_key, template_data), 200 From f3c5289440107e0dd08cfc5039300451238991d8 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Thu, 6 Jun 2024 16:33:54 -0400 Subject: [PATCH 5/7] Correct errors in function docstrings (cherry picked from commit 6ad3b52853b2a24bb85087a0aa73541d01f909ea) --- src/bos/server/controllers/v2/sessiontemplates.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/bos/server/controllers/v2/sessiontemplates.py b/src/bos/server/controllers/v2/sessiontemplates.py index 0d0dbb00..99495da6 100644 --- a/src/bos/server/controllers/v2/sessiontemplates.py +++ b/src/bos/server/controllers/v2/sessiontemplates.py @@ -100,10 +100,8 @@ def put_v2_sessiontemplate(session_template_id): # noqa: E501 try: # Convert the JSON request data into a SessionTemplate object. # Any exceptions caught here would be generated from the model - # (i.e. bos.server.models.session_template). - # An example is an exception for a session template name that - # does not conform to Kubernetes naming convention. - # In this case return 400 with a description of the specific error. + # (i.e. bos.server.models.v2_session_template). + # In that case, return 400 with a description of the specific error. SessionTemplate.from_dict(template_data) except Exception as err: LOGGER.error("Error creating session template: %s", exc_type_msg(err)) @@ -215,10 +213,8 @@ def patch_v2_sessiontemplate(session_template_id): try: # Convert the JSON request data into a SessionTemplate object. # Any exceptions caught here would be generated from the model - # (i.e. bos.server.models.session_template). - # An example is an exception for a session template name that - # does not confirm to Kubernetes naming convention. - # In this case return 400 with a description of the specific error. + # (i.e. bos.server.models.v2_session_template). + # In that case, return 400 with a description of the specific error. SessionTemplate.from_dict(template_data) except Exception as err: LOGGER.error("Error patching session template: %s", exc_type_msg(err)) From cb80903238b9e3254ad0f37bba77cb97a280a155 Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Mon, 29 Jul 2024 16:08:44 -0400 Subject: [PATCH 6/7] CASMCMS-9067: Add session_limit_required option --- CHANGELOG.md | 6 ++++++ api/openapi.yaml.in | 10 ++++++++-- src/bos/operators/utils/clients/bos/options.py | 3 +++ src/bos/server/controllers/v2/options.py | 3 ++- src/bos/server/controllers/v2/sessions.py | 8 ++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 207628b9..db81793e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- New BOS v2 option `session_limit_required` + - If set, new sessions must have a limit specified + - When creating a BOS session, specifying a limit value of `*` has the effect of not limiting + the session, which is one way to create a non-limited session if `session_limit_required` is set. + ### Changed - List Python packages after installing, for build log purposes diff --git a/api/openapi.yaml.in b/api/openapi.yaml.in index 18464202..32126d37 100644 --- a/api/openapi.yaml.in +++ b/api/openapi.yaml.in @@ -363,6 +363,8 @@ components: will be limited. Components are treated as OR operations unless preceded by "&" for AND or "!" for NOT. + Alternatively, the limit can be set to "*", which means no limit. + It is recommended that this should be 1-65535 characters in length. This restriction is not enforced in this version of BOS, but it is @@ -517,7 +519,8 @@ components: Shutdown Power down Components that are on. V2SessionCreate: description: | - A Session Creation object. A UUID name is generated if a name is not provided. + A Session Creation object. A UUID name is generated if a name is not provided. The limit parameter is + required if the session_limit_required option is true. type: object properties: name: @@ -1049,10 +1052,13 @@ components: maximum: 1048576 max_component_batch_size: type: integer - description: The maximum number of components that a BOS operator will process at once. 0 means no limit. + description: The maximum number of Components that a BOS operator will process at once. 0 means no limit. example: 1000 minimum: 0 maximum: 131071 + session_limit_required: + type: boolean + description: If true, Sessions cannot be created without specifying the limit parameter. additionalProperties: true requestBodies: diff --git a/src/bos/operators/utils/clients/bos/options.py b/src/bos/operators/utils/clients/bos/options.py index 3440facb..5990c5b9 100644 --- a/src/bos/operators/utils/clients/bos/options.py +++ b/src/bos/operators/utils/clients/bos/options.py @@ -120,5 +120,8 @@ def default_retry_policy(self): def max_component_batch_size(self): return self.get_option('max_component_batch_size', int, 2800) + @property + def session_limit_required(self): + return self.get_option('session_limit_required', bool, False) options = Options() diff --git a/src/bos/server/controllers/v2/options.py b/src/bos/server/controllers/v2/options.py index 35d98fae..6db6f701 100644 --- a/src/bos/server/controllers/v2/options.py +++ b/src/bos/server/controllers/v2/options.py @@ -49,7 +49,8 @@ 'max_power_off_wait_time': 300, 'polling_frequency': 15, 'default_retry_policy': 3, - 'max_component_batch_size': 2800 + 'max_component_batch_size': 2800, + "session_limit_required": False } diff --git a/src/bos/server/controllers/v2/sessions.py b/src/bos/server/controllers/v2/sessions.py index ff9bb7eb..30a61d6b 100644 --- a/src/bos/server/controllers/v2/sessions.py +++ b/src/bos/server/controllers/v2/sessions.py @@ -36,6 +36,7 @@ from bos.common.values import Phase, Status from bos.server import redis_db_utils as dbutils from bos.server.controllers.v2.components import get_v2_components_data +from bos.server.controllers.v2.options import get_v2_options_data from bos.server.controllers.v2.sessiontemplates import get_v2_sessiontemplate from bos.server.models.v2_session import V2Session as Session # noqa: E501 from bos.server.models.v2_session_create import V2SessionCreate as SessionCreate # noqa: E501 @@ -70,6 +71,13 @@ def post_v2_session(): # noqa: E501 msg = "Post must be in JSON format" LOGGER.error(msg) return msg, 400 + + # If no limit is specified, check to see if we require one + if not session_create.limit and get_v2_options_data().get('session_limit_required', False): + msg = "session_limit_required option is set, but this session has no limit specified" + LOGGER.error(msg) + return msg, 400 + template_name = session_create.template_name LOGGER.debug("Template Name: %s operation: %s", template_name, session_create.operation) # Check that the template_name exists. From 03ee1bf268eadfa248a391537afaa7371191d5da Mon Sep 17 00:00:00 2001 From: "Mitch Harding (the weird one)" Date: Tue, 30 Jul 2024 11:47:42 -0400 Subject: [PATCH 7/7] Release 2.23.0 for CSM 1.6 --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db81793e..b3d088ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.23.0] - 2024-07-30 ### Added - New BOS v2 option `session_limit_required` - If set, new sessions must have a limit specified @@ -23,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Dependencies - Pin major/minor versions for Python packages, but use latest patch version -## [2.22.0] +## [2.22.0] - 2024-07-23 ### Added - Add request timeouts to BOS reporter API calls - Create new BOS v2 `max_component_batch_size` option to limit number of components a BOS operator @@ -39,7 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - Removed redundant `duration_to_timedelta` function definition from BOS reporter source. -## [2.21.0] +## [2.21.0] - 2024-07-12 ### Fixed - The applystage operation works again. It was broken when multi-tenancy support was added.