From a105150a7ea819059df337b412cb1895148253ca Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 09:14:45 -0700 Subject: [PATCH 01/31] RF: Use usermod instead of addgroup in the Dockerfile template. Hopefully closes #305. --- cloudknot/templates/Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index cf1b8d8b..622717ec 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -14,7 +14,7 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt${github_installs_string} # Add user to "staff" group. # Give user a home directory. RUN (id -u ${username} >/dev/null 2>&1 || useradd ${username}) \ - && addgroup ${username} staff \ + && usermod -a -G staff "${username}" \ && mkdir -p /home/${username} \ && chown -R ${username}:staff /home/${username} From a0912a87389a7c3f53f5198f9962871f6e9f1c8d Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 09:17:13 -0700 Subject: [PATCH 02/31] Apply change of adduser to usermod in the test Dockerfiles as well. --- cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile | 2 +- cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile b/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile index 5b075334..7fc7b6d3 100644 --- a/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile +++ b/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile @@ -14,7 +14,7 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt # Add user to "staff" group. # Give user a home directory. RUN (id -u cloudknot-user >/dev/null 2>&1 || useradd cloudknot-user) \ - && addgroup cloudknot-user staff \ + && usermod -a -G staff "cloudknot-user" \ && mkdir -p /home/cloudknot-user \ && chown -R cloudknot-user:staff /home/cloudknot-user diff --git a/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile b/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile index f332d1c3..49005c4c 100644 --- a/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile +++ b/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile @@ -14,7 +14,7 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt # Add user to "staff" group. # Give user a home directory. RUN (id -u unit-test-username >/dev/null 2>&1 || useradd unit-test-username) \ - && addgroup unit-test-username staff \ + && usermod -a -G staff "${unit-test-username}" \ && mkdir -p /home/unit-test-username \ && chown -R unit-test-username:staff /home/unit-test-username From 1fcdcf57fa1bb604ae5acc4b579b38d1287862da Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 09:18:17 -0700 Subject: [PATCH 03/31] Removes testing on older Python version (3.6 and 3.7) And also add testing on newer 3.10. --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 47a58342..37e6154e 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 8 matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: ["3.8", "3.9", "3.10"] steps: - name: Checkout repo From dd33e9c797f0d27f4fec7c7803797c3393a6b40b Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 09:19:45 -0700 Subject: [PATCH 04/31] Build the docs only on 3.9 --- .github/workflows/docbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml index e57d9a00..cd600882 100644 --- a/.github/workflows/docbuild.yml +++ b/.github/workflows/docbuild.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: 3.9 steps: - name: Checkout repo From e315ae13584f2ff0c34c15c0d7305b4bb38b0bf8 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 09:20:47 -0700 Subject: [PATCH 05/31] Update call to pip installation. --- .github/workflows/pythonpackage.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 37e6154e..2078256d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -19,9 +19,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install software run: | - python -m pip install --upgrade pip --use-feature=2020-resolver - python -m pip install coveralls --use-feature=2020-resolver - python -m pip install .[dev] --use-feature=2020-resolver + python -m pip install --upgrade pip + python -m pip install coveralls + python -m pip install .[dev] python -m pip install https://github.com/bboe/coveralls-python/archive/github_actions.zip - name: Configure run: | From 535b6561b788fbafee47d265f1a21f8f3e06b195 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 09:21:26 -0700 Subject: [PATCH 06/31] Build docs on both 3.9 and 3.10. --- .github/workflows/docbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml index cd600882..4bf89302 100644 --- a/.github/workflows/docbuild.yml +++ b/.github/workflows/docbuild.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: 3.9 + python-version: ["3.9", "3.10"] steps: - name: Checkout repo From 44aa104e882300ce92e6f20ce8775434d8947de1 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 10:46:26 -0700 Subject: [PATCH 07/31] Update black/pre-commit configuration/versions. --- .pre-commit-config.yaml | 3 +-- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a30267f3..c25c21a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,5 @@ repos: - repo: https://github.com/python/black - rev: 22.1.0 + rev: 23.7.0 hooks: - id: black - language_version: python3.9 diff --git a/setup.cfg b/setup.cfg index 86c839b2..5f809e5b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,7 +59,7 @@ dev = numpydoc==1.1.0 moto>=2.2.13.dev2 responses==0.12.0 - pre-commit==2.9.2 + pre-commit==3.3.3 pydocstyle==5.1.1 pytest-cov==2.10.1 pytest-xdist[psutil]==2.1.0 From 9fd2907741b8b15d72fcadf73d053068454780c7 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:05:20 -0700 Subject: [PATCH 08/31] More compatibility issues --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5f809e5b..b0f74913 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,8 +57,7 @@ dev = coverage==5.3 flake8==3.8.3 numpydoc==1.1.0 - moto>=2.2.13.dev2 - responses==0.12.0 + moto==4.1.13 pre-commit==3.3.3 pydocstyle==5.1.1 pytest-cov==2.10.1 From 9b760da75de3b5d860d9b65e219cbfe7b5773ca4 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:10:41 -0700 Subject: [PATCH 09/31] Update deprecated imports from the collections module. Python 3.10 compatibility. --- cloudknot/cloudknot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cloudknot/cloudknot.py b/cloudknot/cloudknot.py index 92517356..38da5fa3 100644 --- a/cloudknot/cloudknot.py +++ b/cloudknot/cloudknot.py @@ -5,10 +5,9 @@ import logging import os -try: - from collections.abc import Iterable, namedtuple -except ImportError: - from collections import Iterable, namedtuple +from collections.abc import Iterable +from collections import namedtuple + from concurrent.futures import ThreadPoolExecutor from . import aws From 5c45502b4d3db5bbbb88486454d1a0c7adc518c3 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:16:18 -0700 Subject: [PATCH 10/31] Upgrade black --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index b0f74913..76bffb81 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,7 +53,7 @@ console_scripts = [options.extras_require] dev = - black==20.8b1 + black==23.7.0 coverage==5.3 flake8==3.8.3 numpydoc==1.1.0 From 06ec1cd54cc48b77113012a378af7e629a84e2c5 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:20:02 -0700 Subject: [PATCH 11/31] Don't flake8 these files. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 76bffb81..3d9a7404 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,4 +76,4 @@ match-dir = cloudknot max-line-length = 88 select = C,E,F,W,B,B950 ignore = E501,N802,N806,W503,E203 -exclude = setup.py,build,dist,doc,examples,cloudknot/data +exclude = setup.py,build,dist,doc,examples,cloudknot/data,cloudknot/_version,cloudknot/cli From ffc876bd058a64a41d7d44b2174344fd8478ac18 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:27:29 -0700 Subject: [PATCH 12/31] Maybe we do need the extensions for these two files. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3d9a7404..db01feb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,4 +76,4 @@ match-dir = cloudknot max-line-length = 88 select = C,E,F,W,B,B950 ignore = E501,N802,N806,W503,E203 -exclude = setup.py,build,dist,doc,examples,cloudknot/data,cloudknot/_version,cloudknot/cli +exclude = setup.py,build,dist,doc,examples,cloudknot/data,cloudknot/_version.py,cloudknot/cli.py From 14a787451bcaf8eb09a4f8d7ab92aa9521bacbbe Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:32:16 -0700 Subject: [PATCH 13/31] Exclude these directly on the CI. --- .github/workflows/pythonpackage.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 2078256d..6518dd52 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -30,7 +30,7 @@ jobs: printf "[aws]\nconfigured = True\n" > ~/.aws/cloudknot - name: Lint with flake8 run: | - flake8 + flake8 --exclude /home/runner/work/cloudknot/cloudknot/cloudknot/_version.py, /home/runner/work/cloudknot/cloudknot/cloudknot/cli.py black --check . pydocstyle - name: Test diff --git a/setup.cfg b/setup.cfg index db01feb8..76bffb81 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,4 +76,4 @@ match-dir = cloudknot max-line-length = 88 select = C,E,F,W,B,B950 ignore = E501,N802,N806,W503,E203 -exclude = setup.py,build,dist,doc,examples,cloudknot/data,cloudknot/_version.py,cloudknot/cli.py +exclude = setup.py,build,dist,doc,examples,cloudknot/data From 39c392dae6b6505d51fdc9226f05d9afa826f822 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:36:15 -0700 Subject: [PATCH 14/31] Turns out this is not a flake8 thing, but a black thing. So there you have it. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6d944aa1..9d2a6abb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ exclude = ''' | build | dist )/ - | foo.py # also separately exclude a file named foo.py in - # the root of the project + | _version.py + | cli.py ) ''' From 552a35f2ad56d13d33f68c11fdd9a65ecdff1c6d Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:38:48 -0700 Subject: [PATCH 15/31] Update pytest. May help with https://stackoverflow.com/a/69569206 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 76bffb81..70253aef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,7 +62,7 @@ dev = pydocstyle==5.1.1 pytest-cov==2.10.1 pytest-xdist[psutil]==2.1.0 - pytest==6.0.1 + pytest==7.4.0 sphinx==3.2.1 maint = rapidfuzz==0.12.2 From 522237720a2f6b05bd81755b722332c1a2fe4281 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:43:45 -0700 Subject: [PATCH 16/31] Install moto with the optional cloudformation dependencies. We're hardcore. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 70253aef..5d793224 100644 --- a/setup.cfg +++ b/setup.cfg @@ -57,7 +57,7 @@ dev = coverage==5.3 flake8==3.8.3 numpydoc==1.1.0 - moto==4.1.13 + moto[cloudformation]==4.1.13 pre-commit==3.3.3 pydocstyle==5.1.1 pytest-cov==2.10.1 From f4e695e6d01b5422f8f4d999f3487fcddcf14020 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 14:47:34 -0700 Subject: [PATCH 17/31] Update to newer sphinx version. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 5d793224..3f767cc0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,7 @@ dev = pytest-cov==2.10.1 pytest-xdist[psutil]==2.1.0 pytest==7.4.0 - sphinx==3.2.1 + sphinx==7.1.1 maint = rapidfuzz==0.12.2 From 5854dd5dfbc2e9b0e0dba11ff3da8b15525a0bdb Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 15:15:35 -0700 Subject: [PATCH 18/31] Replace underscore in username with dash, so that bucket names are S3-allowable --- cloudknot/aws/base_classes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudknot/aws/base_classes.py b/cloudknot/aws/base_classes.py index 620fb91f..42bf7b5f 100644 --- a/cloudknot/aws/base_classes.py +++ b/cloudknot/aws/base_classes.py @@ -669,6 +669,7 @@ def get_user(): username = user_info.get("UserName") if username is None: username = user_info.get("Arn").split(":")[-1] + username = username.replace("_", "-") return username From 5d193a7a7b7b3207923f961fe7170a98ec5d7b83 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 15:22:01 -0700 Subject: [PATCH 19/31] Replace right before setting the bucket name --- cloudknot/aws/base_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudknot/aws/base_classes.py b/cloudknot/aws/base_classes.py index 42bf7b5f..6896fd63 100644 --- a/cloudknot/aws/base_classes.py +++ b/cloudknot/aws/base_classes.py @@ -251,6 +251,7 @@ def get_s3_params(): # Use set_s3_params to check for name availability # and write to config file + bucket = bucket.replace("_", "-") # S3 does not allow underscores set_s3_params(bucket=bucket, policy=policy, sse=sse) if policy is None: @@ -669,7 +670,6 @@ def get_user(): username = user_info.get("UserName") if username is None: username = user_info.get("Arn").split(":")[-1] - username = username.replace("_", "-") return username From 32af62e6e657f713b8baf18377c10013a05aa54a Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 20:03:05 -0700 Subject: [PATCH 20/31] Find a combination of awscli and sphinx versions that works --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3f767cc0..43b2b3d7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,8 +33,8 @@ setup_requires = setuptools_scm python_requires = >=3.6 install_requires = - awscli boto3>=1.5.21 + awscli>=1.28.0 botocore>=1.8.36 cloudpickle docker>=3.0.0 @@ -63,7 +63,7 @@ dev = pytest-cov==2.10.1 pytest-xdist[psutil]==2.1.0 pytest==7.4.0 - sphinx==7.1.1 + sphinx==4.5.0 maint = rapidfuzz==0.12.2 From 7daf3f193e0a4d5c016d802b6008818f0408bed9 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 20:12:59 -0700 Subject: [PATCH 21/31] Try adding a shebang to the template. Otherwise running into 'exec format error' on Batch --- cloudknot/templates/script.template | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudknot/templates/script.template b/cloudknot/templates/script.template index 6383ff11..5d7068c5 100644 --- a/cloudknot/templates/script.template +++ b/cloudknot/templates/script.template @@ -1,3 +1,4 @@ +#!/usr/bin/python import boto3 import cloudpickle import os From 782bce7ff18b3c1c3cafdf166f7dd3ad20eb30b9 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 20:38:46 -0700 Subject: [PATCH 22/31] use CMD instead of ENTRYPOINT --- cloudknot/templates/Dockerfile.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index 622717ec..2a14b9e5 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -23,8 +23,8 @@ ENV HOME /home/${username} # Set working directory WORKDIR /home/${username} -# Set entrypoint -ENTRYPOINT ["python", "/home/${username}/${script_base_name}"] +# Set command to run +CMD ["python", "/home/${username}/${script_base_name}"] # Copy the python script COPY ${script_base_name} /home/${username}/ From 0d3415dff01be6c2ba332e3cedf1328ac1a2f155 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 20:48:29 -0700 Subject: [PATCH 23/31] Do the normal thing again --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 6518dd52..2078256d 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -30,7 +30,7 @@ jobs: printf "[aws]\nconfigured = True\n" > ~/.aws/cloudknot - name: Lint with flake8 run: | - flake8 --exclude /home/runner/work/cloudknot/cloudknot/cloudknot/_version.py, /home/runner/work/cloudknot/cloudknot/cloudknot/cli.py + flake8 black --check . pydocstyle - name: Test From 29339e7ed61ba096a80467ee3a792800873cad1a Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 20:49:02 -0700 Subject: [PATCH 24/31] Use ENTRYPOINT, but be explicit about the python to call --- cloudknot/templates/Dockerfile.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index 2a14b9e5..4278b172 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -23,8 +23,8 @@ ENV HOME /home/${username} # Set working directory WORKDIR /home/${username} -# Set command to run -CMD ["python", "/home/${username}/${script_base_name}"] +# Set entrypoint +ENTRYPOINT ["/usr/bin/python", "/home/${username}/${script_base_name}"] # Copy the python script COPY ${script_base_name} /home/${username}/ From 8f7051eb13329ac3a22898fcb9fda0324a8b63f9 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Sat, 29 Jul 2023 21:08:09 -0700 Subject: [PATCH 25/31] User call directly to Python again as instructed in python docker docs. --- cloudknot/templates/Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index 4278b172..622717ec 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -24,7 +24,7 @@ ENV HOME /home/${username} WORKDIR /home/${username} # Set entrypoint -ENTRYPOINT ["/usr/bin/python", "/home/${username}/${script_base_name}"] +ENTRYPOINT ["python", "/home/${username}/${script_base_name}"] # Copy the python script COPY ${script_base_name} /home/${username}/ From 9627abc99dc93ea18676886d106f2a9683bd457d Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 29 Jan 2024 15:15:51 -0800 Subject: [PATCH 26/31] Upgrade sphinx version. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 43b2b3d7..6c510043 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,7 +63,7 @@ dev = pytest-cov==2.10.1 pytest-xdist[psutil]==2.1.0 pytest==7.4.0 - sphinx==4.5.0 + sphinx==5.0.0 maint = rapidfuzz==0.12.2 From 121d076b541969675958460183d0edf2152cc34d Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 31 Jan 2024 13:30:12 -0800 Subject: [PATCH 27/31] Set "moto[cloudformation]==4.1.5" to address failures in test_docker_image and test_knot --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6c510043..cc361d29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,9 +33,9 @@ setup_requires = setuptools_scm python_requires = >=3.6 install_requires = - boto3>=1.5.21 - awscli>=1.28.0 - botocore>=1.8.36 + boto3>=1.34.31 + awscli>=1.32.31 + botocore>=1.34.31 cloudpickle docker>=3.0.0 pipreqs>=0.4.11 @@ -57,7 +57,7 @@ dev = coverage==5.3 flake8==3.8.3 numpydoc==1.1.0 - moto[cloudformation]==4.1.13 + moto[cloudformation]==4.1.5 pre-commit==3.3.3 pydocstyle==5.1.1 pytest-cov==2.10.1 From ab2b0eadf1eb69071659ad3d16084cf8cc536c85 Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 31 Jan 2024 13:32:26 -0800 Subject: [PATCH 28/31] Updated Dockerfile template to address #305 --- .../docker_reqs_ref_data/py3/ref1/Dockerfile | 19 +++++++++---------- .../docker_reqs_ref_data/py3/ref2/Dockerfile | 19 +++++++++---------- cloudknot/templates/Dockerfile.template | 19 +++++++++---------- 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile b/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile index 7fc7b6d3..6f0dc50e 100644 --- a/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile +++ b/cloudknot/data/docker_reqs_ref_data/py3/ref1/Dockerfile @@ -13,18 +13,17 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt # Create a default user. Available via runtime flag `--user cloudknot-user`. # Add user to "staff" group. # Give user a home directory. -RUN (id -u cloudknot-user >/dev/null 2>&1 || useradd cloudknot-user) \ - && usermod -a -G staff "cloudknot-user" \ - && mkdir -p /home/cloudknot-user \ - && chown -R cloudknot-user:staff /home/cloudknot-user +RUN groupadd -f staff \ + && useradd --create-home --groups staff "cloudknot-user" -ENV HOME /home/cloudknot-user +# Copy the python script +COPY --chown="cloudknot-user" "unit-testing-func.py" "/home/cloudknot-user/" + +# Set user +USER "cloudknot-user" # Set working directory -WORKDIR /home/cloudknot-user +WORKDIR "/home/cloudknot-user" # Set entrypoint -ENTRYPOINT ["python", "/home/cloudknot-user/unit-testing-func.py"] - -# Copy the python script -COPY unit-testing-func.py /home/cloudknot-user/ +ENTRYPOINT ["/home/cloudknot-user/unit-testing-func.py"] diff --git a/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile b/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile index 49005c4c..d4cbfe4d 100644 --- a/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile +++ b/cloudknot/data/docker_reqs_ref_data/py3/ref2/Dockerfile @@ -13,18 +13,17 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt # Create a default user. Available via runtime flag `--user unit-test-username`. # Add user to "staff" group. # Give user a home directory. -RUN (id -u unit-test-username >/dev/null 2>&1 || useradd unit-test-username) \ - && usermod -a -G staff "${unit-test-username}" \ - && mkdir -p /home/unit-test-username \ - && chown -R unit-test-username:staff /home/unit-test-username +RUN groupadd -f staff \ + && useradd --create-home --groups staff "unit-test-username" -ENV HOME /home/unit-test-username +# Copy the python script +COPY --chown="unit-test-username" "test-func-input.py" "/home/unit-test-username/" + +# Set user +USER "unit-test-username" # Set working directory -WORKDIR /home/unit-test-username +WORKDIR "/home/unit-test-username" # Set entrypoint -ENTRYPOINT ["python", "/home/unit-test-username/test-func-input.py"] - -# Copy the python script -COPY test-func-input.py /home/unit-test-username/ +ENTRYPOINT ["/home/unit-test-username/test-func-input.py"] diff --git a/cloudknot/templates/Dockerfile.template b/cloudknot/templates/Dockerfile.template index 622717ec..259aa926 100644 --- a/cloudknot/templates/Dockerfile.template +++ b/cloudknot/templates/Dockerfile.template @@ -13,18 +13,17 @@ RUN pip install --no-cache-dir -r /tmp/requirements.txt${github_installs_string} # Create a default user. Available via runtime flag `--user ${username}`. # Add user to "staff" group. # Give user a home directory. -RUN (id -u ${username} >/dev/null 2>&1 || useradd ${username}) \ - && usermod -a -G staff "${username}" \ - && mkdir -p /home/${username} \ - && chown -R ${username}:staff /home/${username} +RUN groupadd -f staff \ + && useradd --create-home --groups staff "${username}" -ENV HOME /home/${username} +# Copy the python script +COPY --chown="${username}" "${script_base_name}" "/home/${username}/" + +# Set user +USER "${username}" # Set working directory -WORKDIR /home/${username} +WORKDIR "/home/${username}" # Set entrypoint -ENTRYPOINT ["python", "/home/${username}/${script_base_name}"] - -# Copy the python script -COPY ${script_base_name} /home/${username}/ +ENTRYPOINT ["/home/${username}/${script_base_name}"] From 4242722bbd0d53da35028acf40a13c23ec21296d Mon Sep 17 00:00:00 2001 From: Altan Orhon Date: Wed, 31 Jan 2024 13:35:14 -0800 Subject: [PATCH 29/31] Update shebang to use `env` and ensure presence in data files --- .../data/docker_reqs_ref_data/py3/ref1/unit-testing-func.py | 1 + cloudknot/data/docker_reqs_ref_data/py3/ref2/test-func-input.py | 1 + cloudknot/templates/script.template | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudknot/data/docker_reqs_ref_data/py3/ref1/unit-testing-func.py b/cloudknot/data/docker_reqs_ref_data/py3/ref1/unit-testing-func.py index 55ddaa27..ecaeb66b 100644 --- a/cloudknot/data/docker_reqs_ref_data/py3/ref1/unit-testing-func.py +++ b/cloudknot/data/docker_reqs_ref_data/py3/ref1/unit-testing-func.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import boto3 import cloudpickle import os diff --git a/cloudknot/data/docker_reqs_ref_data/py3/ref2/test-func-input.py b/cloudknot/data/docker_reqs_ref_data/py3/ref2/test-func-input.py index 55ddaa27..ecaeb66b 100644 --- a/cloudknot/data/docker_reqs_ref_data/py3/ref2/test-func-input.py +++ b/cloudknot/data/docker_reqs_ref_data/py3/ref2/test-func-input.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import boto3 import cloudpickle import os diff --git a/cloudknot/templates/script.template b/cloudknot/templates/script.template index 5d7068c5..d885e2ed 100644 --- a/cloudknot/templates/script.template +++ b/cloudknot/templates/script.template @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import boto3 import cloudpickle import os From 2d9a1beb56debfa6ba580c555d8ede7684268ff5 Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 26 Feb 2024 09:42:29 -0800 Subject: [PATCH 30/31] Update .github/workflows/docbuild.yml --- .github/workflows/docbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docbuild.yml b/.github/workflows/docbuild.yml index 4bf89302..81b6d512 100644 --- a/.github/workflows/docbuild.yml +++ b/.github/workflows/docbuild.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.10", "3.11"] steps: - name: Checkout repo From ef81704693dbfcc26312cf1491145296d42e1dae Mon Sep 17 00:00:00 2001 From: Ariel Rokem Date: Mon, 26 Feb 2024 09:42:37 -0800 Subject: [PATCH 31/31] Update .github/workflows/pythonpackage.yml --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 2078256d..9dd97c85 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -8,7 +8,7 @@ jobs: strategy: max-parallel: 8 matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.10", "3.11"] steps: - name: Checkout repo