From fd5f6ad7f70b8bf814e92594615a31515fee4f77 Mon Sep 17 00:00:00 2001 From: Tom Queen Date: Thu, 19 Sep 2024 11:41:46 +0100 Subject: [PATCH 01/11] Update README.md From 0238b0c82f59109f11dd5a4c2c4e882412f61f2c Mon Sep 17 00:00:00 2001 From: Tom Queen Date: Thu, 19 Sep 2024 12:17:44 +0100 Subject: [PATCH 02/11] Update README.md From 22b64197cc5208d9d1e3917a9c4280a0d19fe3a4 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 12:32:28 +0100 Subject: [PATCH 03/11] start trying to add jammy checks --- docker/molecule/base/jammy/Dockerfile | 60 ++++++++++++++++++ docker/molecule/base/jammy/README.md | 1 + docker/molecule/base/jammy/buildspec.yml | 51 +++++++++++++++ .../base/jammy/docker-compose.test.yml | 28 +++++++++ docker/molecule/base/jammy/hooks/build | 8 +++ docker/molecule/devel/jammy/Dockerfile | 37 +++++++++++ docker/molecule/devel/jammy/README.md | 9 +++ docker/molecule/devel/jammy/buildspec.yml | 52 +++++++++++++++ .../devel/jammy/docker-compose.test.yml | 28 +++++++++ docker/molecule/devel/jammy/hooks/build | 8 +++ docker/molecule/docker/jammy/Dockerfile | 28 +++++++++ docker/molecule/docker/jammy/README.md | 1 + docker/molecule/docker/jammy/buildspec.yml | 63 +++++++++++++++++++ .../docker/jammy/docker-compose.test.yml | 25 ++++++++ docker/molecule/docker/jammy/hooks/build | 8 +++ docker/molecule/ec2/jammy/Dockerfile | 32 ++++++++++ docker/molecule/ec2/jammy/README.md | 1 + docker/molecule/ec2/jammy/buildspec.yml | 52 +++++++++++++++ .../ec2/jammy/docker-compose.test.yml | 25 ++++++++ docker/molecule/ec2/jammy/hooks/build | 8 +++ docker/test/docker/jammy/Dockerfile | 38 +++++++++++ docker/test/docker/jammy/buildspec.yml | 63 +++++++++++++++++++ .../test/docker/jammy/docker-compose.test.yml | 25 ++++++++ docker/test/docker/jammy/hooks/build | 8 +++ docker/test/ubuntu/jammy/Dockerfile | 31 +++++++++ docker/test/ubuntu/jammy/buildspec.yml | 52 +++++++++++++++ .../test/ubuntu/jammy/docker-compose.test.yml | 20 ++++++ 27 files changed, 762 insertions(+) create mode 100644 docker/molecule/base/jammy/Dockerfile create mode 100644 docker/molecule/base/jammy/README.md create mode 100644 docker/molecule/base/jammy/buildspec.yml create mode 100644 docker/molecule/base/jammy/docker-compose.test.yml create mode 100755 docker/molecule/base/jammy/hooks/build create mode 100644 docker/molecule/devel/jammy/Dockerfile create mode 100644 docker/molecule/devel/jammy/README.md create mode 100644 docker/molecule/devel/jammy/buildspec.yml create mode 100644 docker/molecule/devel/jammy/docker-compose.test.yml create mode 100755 docker/molecule/devel/jammy/hooks/build create mode 100644 docker/molecule/docker/jammy/Dockerfile create mode 100644 docker/molecule/docker/jammy/README.md create mode 100644 docker/molecule/docker/jammy/buildspec.yml create mode 100644 docker/molecule/docker/jammy/docker-compose.test.yml create mode 100755 docker/molecule/docker/jammy/hooks/build create mode 100644 docker/molecule/ec2/jammy/Dockerfile create mode 100644 docker/molecule/ec2/jammy/README.md create mode 100644 docker/molecule/ec2/jammy/buildspec.yml create mode 100644 docker/molecule/ec2/jammy/docker-compose.test.yml create mode 100755 docker/molecule/ec2/jammy/hooks/build create mode 100644 docker/test/docker/jammy/Dockerfile create mode 100644 docker/test/docker/jammy/buildspec.yml create mode 100644 docker/test/docker/jammy/docker-compose.test.yml create mode 100755 docker/test/docker/jammy/hooks/build create mode 100644 docker/test/ubuntu/jammy/Dockerfile create mode 100644 docker/test/ubuntu/jammy/buildspec.yml create mode 100644 docker/test/ubuntu/jammy/docker-compose.test.yml diff --git a/docker/molecule/base/jammy/Dockerfile b/docker/molecule/base/jammy/Dockerfile new file mode 100644 index 000000000..dc9ea2aa6 --- /dev/null +++ b/docker/molecule/base/jammy/Dockerfile @@ -0,0 +1,60 @@ +FROM public.ecr.aws/shadowrobot/ubuntu:jammy + +MAINTAINER "Shadow Robot's Software Team " + +LABEL Description="Ubuntu Jammy image contains Ansible and Molecule" Vendor="Shadow Robot" Version="1.0" + +ENV DEBIAN_FRONTEND noninteractive + +ENV LC_ALL C.UTF-8 + +ENV LANG C.UTF-8 + +ARG ml_base_aurora_branch + +ENV ml_base_aurora_branch ${ml_base_aurora_branch:-"master"} + +# TODO BEFORE MERGE - check setuptools==51.1.1 + +RUN set -x && \ + apt-get update && \ + \ + echo "Installing wget" && \ + apt-get install -y --no-install-recommends ca-certificates wget && \ + \ + echo "Installing dependencies for python3.8, SSL, pip3" && \ + apt-get install -y build-essential zlib1g-dev libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \ + \ + echo "Installing git, sudo and Python 3 pip and dev" && \ + apt-get install -y \ + python3-dev \ + python3-pip \ + git \ + libyaml-dev \ + python-crypto \ + libssl-dev \ + libffi-dev \ + sshpass \ + sudo && \ + \ + pip3 install --upgrade pip setuptools==51.1.1 gcovr && \ + echo "Cloning Aurora" && \ + git clone --depth 1 https://github.com/shadow-robot/aurora.git -b $ml_base_aurora_branch /tmp/aurora && \ + \ + pip3 --version && \ + python3 --version && \ + echo "Installing molecule" && \ + pip3 install -r /tmp/aurora/ansible/data/molecule/requirements.txt && \ + \ + echo "Clean up" && \ + apt-get clean && \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /root/.ansible \ + /root/.gitconfig \ + /root/.cache + +VOLUME /var/lib/docker + diff --git a/docker/molecule/base/jammy/README.md b/docker/molecule/base/jammy/README.md new file mode 100644 index 000000000..16625bc39 --- /dev/null +++ b/docker/molecule/base/jammy/README.md @@ -0,0 +1 @@ +Please read [these instructions](../../../../docs/molecule_dockers.md). diff --git a/docker/molecule/base/jammy/buildspec.yml b/docker/molecule/base/jammy/buildspec.yml new file mode 100644 index 000000000..a0ebcdf5b --- /dev/null +++ b/docker/molecule/base/jammy/buildspec.yml @@ -0,0 +1,51 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: 0.2 +env: + shell: bash + variables: + branch: master + image: aurora-molecule + tag: focal + dockerfile_folder: docker/molecule/base/focal/ + exported-variables: + - branch + - image + - tag + - dockerfile_folder +phases: + install: + runtime-versions: + docker: 19 + build: + commands: + - cd .. + - rm -rf aurora + - git clone https://github.com/shadow-robot/aurora.git + - cd aurora + - git checkout $branch + - awscli_v1_executable=$(which aws) + - rm $awscli_v1_executable + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - cd $dockerfile_folder + - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot + - docker build -t $image . + - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag + - docker push public.ecr.aws/shadowrobot/$image:$tag + - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt + - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi diff --git a/docker/molecule/base/jammy/docker-compose.test.yml b/docker/molecule/base/jammy/docker-compose.test.yml new file mode 100644 index 000000000..8bfbd4de0 --- /dev/null +++ b/docker/molecule/base/jammy/docker-compose.test.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: '3' + +services: + sut: + build: + context: . + args: + ml_base_aurora_branch: ${SOURCE_BRANCH} + command: > + bash -c "pip3 freeze | grep molecule && + pip3 freeze | grep ansible && + pip3 freeze | grep docker" + environment: + - SOURCE_BRANCH diff --git a/docker/molecule/base/jammy/hooks/build b/docker/molecule/base/jammy/hooks/build new file mode 100755 index 000000000..37955d557 --- /dev/null +++ b/docker/molecule/base/jammy/hooks/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +export DOCKERFILE_PATH=${DOCKERFILE_PATH:-Dockerfile} + +echo "Executing build command" +echo "docker build --build-arg ml_base_aurora_branch=${SOURCE_BRANCH} -f ${DOCKERFILE_PATH} -t ${IMAGE_NAME} ." + +docker build --build-arg ml_base_aurora_branch="${SOURCE_BRANCH}" -f "${DOCKERFILE_PATH}" -t "${IMAGE_NAME}" . diff --git a/docker/molecule/devel/jammy/Dockerfile b/docker/molecule/devel/jammy/Dockerfile new file mode 100644 index 000000000..341b3eccd --- /dev/null +++ b/docker/molecule/devel/jammy/Dockerfile @@ -0,0 +1,37 @@ +FROM public.ecr.aws/shadowrobot/aurora-molecule-docker:jammy + +LABEL Description="Ubuntu Jammy image contains Ansible, Molecule, Docker and EC2 driver and Development tools with GUI" Vendor="Shadow Robot" Version="1.0" + +ENV DEBIAN_FRONTEND noninteractive + +ENV PULSE_SERVER /run/pulse/native + +ARG ml_devel_aurora_branch + +ENV ml_devel_aurora_branch ${ml_devel_aurora_branch:-"master"} + +ENV ml_devel_aurora_script "https://raw.githubusercontent.com/shadow-robot/aurora/$ml_devel_aurora_branch/bin/run-ansible.sh" + +RUN set -x && \ + \ + echo "Downloading one-liner" && \ + wget -O /tmp/oneliner "$( echo "$ml_devel_aurora_script" | sed 's/#/%23/g' )" && \ + chmod 755 /tmp/oneliner && \ + \ + echo "Setting up UI and new Docker user" && \ + /tmp/oneliner configure_software --branch $ml_devel_aurora_branch software=[setup-docker] && \ + \ + echo "Add user to docker group" && \ + usermod -aG docker user && \ + \ + echo "Installing IDEs and AWS CLI" && \ + /tmp/oneliner install_software --branch $ml_devel_aurora_branch software=[pycharm,vscode,aws-cli] && \ + \ + echo "Clean up" && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.ansible /root/.gitconfig /root/.cache + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +VOLUME /var/lib/docker +CMD ["/usr/bin/terminator"] + diff --git a/docker/molecule/devel/jammy/README.md b/docker/molecule/devel/jammy/README.md new file mode 100644 index 000000000..c850c197e --- /dev/null +++ b/docker/molecule/devel/jammy/README.md @@ -0,0 +1,9 @@ +## Launching container with UI + +In order to launch development image please use the following command + +```bash +docker run -it --name sample_dev -e DISPLAY -e QT_X11_NO_MITSHM=1 -e LOCAL_USER_ID=$(id -u) \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp/.X11-unix:/tmp/.X11-unix:rw public.ecr.aws/shadowrobot/aurora-molecule-devel:jammy +``` diff --git a/docker/molecule/devel/jammy/buildspec.yml b/docker/molecule/devel/jammy/buildspec.yml new file mode 100644 index 000000000..0938a6b9c --- /dev/null +++ b/docker/molecule/devel/jammy/buildspec.yml @@ -0,0 +1,52 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: 0.2 + +env: + shell: bash + variables: + branch: master + image: aurora-molecule-devel + tag: jammy + dockerfile_folder: docker/molecule/devel/jammy/ + exported-variables: + - branch + - image + - tag + - dockerfile_folder +phases: + install: + runtime-versions: + docker: 19 + build: + commands: + - cd .. + - rm -rf aurora + - git clone https://github.com/shadow-robot/aurora.git + - cd aurora + - git checkout $branch + - awscli_v1_executable=$(which aws) + - rm $awscli_v1_executable + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - cd $dockerfile_folder + - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot + - docker build -t $image . + - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag + - docker push public.ecr.aws/shadowrobot/$image:$tag + - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt + - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi diff --git a/docker/molecule/devel/jammy/docker-compose.test.yml b/docker/molecule/devel/jammy/docker-compose.test.yml new file mode 100644 index 000000000..415286d1e --- /dev/null +++ b/docker/molecule/devel/jammy/docker-compose.test.yml @@ -0,0 +1,28 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: '3' + +services: + sut: + build: + context: . + args: + ml_devel_aurora_branch: ${SOURCE_BRANCH} + command: > + bash -c "aws --version && + test -f /usr/bin/pycharm && + test -f /usr/bin/code" + environment: + - SOURCE_BRANCH diff --git a/docker/molecule/devel/jammy/hooks/build b/docker/molecule/devel/jammy/hooks/build new file mode 100755 index 000000000..378505089 --- /dev/null +++ b/docker/molecule/devel/jammy/hooks/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +export DOCKERFILE_PATH=${DOCKERFILE_PATH:-Dockerfile} + +echo "Executing build command" +echo "docker build --build-arg ml_devel_aurora_branch=${SOURCE_BRANCH} -f ${DOCKERFILE_PATH} -t ${IMAGE_NAME} ." + +docker build --build-arg ml_devel_aurora_branch="${SOURCE_BRANCH}" -f "${DOCKERFILE_PATH}" -t "${IMAGE_NAME}" . diff --git a/docker/molecule/docker/jammy/Dockerfile b/docker/molecule/docker/jammy/Dockerfile new file mode 100644 index 000000000..ace6300d7 --- /dev/null +++ b/docker/molecule/docker/jammy/Dockerfile @@ -0,0 +1,28 @@ +FROM public.ecr.aws/shadowrobot/aurora-molecule:jammy + +LABEL Description="Ubuntu Jammy image contains Ansible, Molecule and Docker driver" Vendor="Shadow Robot" Version="1.0" + +ARG ml_docker_aurora_branch + +ENV ml_docker_aurora_branch ${ml_docker_aurora_branch:-"master"} + +ENV ml_docker_aurora_script "https://raw.githubusercontent.com/shadow-robot/aurora/$ml_docker_aurora_branch/bin/run-ansible.sh" + +RUN set -x && \ + \ + echo "Installing Docker" && \ + wget -O /tmp/oneliner "$( echo "$ml_docker_aurora_script" | sed 's/#/%23/g' )" && \ + chmod 755 /tmp/oneliner && \ + /tmp/oneliner install_software --branch $ml_docker_aurora_branch software=[docker] skip_molecule_task=true && \ + \ + echo "Clean up" && \ + apt-get clean && \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /root/.ansible \ + /root/.gitconfig \ + /root/.cache + +VOLUME /var/lib/docker diff --git a/docker/molecule/docker/jammy/README.md b/docker/molecule/docker/jammy/README.md new file mode 100644 index 000000000..16625bc39 --- /dev/null +++ b/docker/molecule/docker/jammy/README.md @@ -0,0 +1 @@ +Please read [these instructions](../../../../docs/molecule_dockers.md). diff --git a/docker/molecule/docker/jammy/buildspec.yml b/docker/molecule/docker/jammy/buildspec.yml new file mode 100644 index 000000000..9c1dacf3d --- /dev/null +++ b/docker/molecule/docker/jammy/buildspec.yml @@ -0,0 +1,63 @@ +# Copyright 2022-2023 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: 0.2 + +env: + shell: bash + variables: + image: aurora-molecule-docker + tag: jammy + dockerfile_folder: docker/molecule/docker/jammy/ + exported-variables: + - image + - tag + - dockerfile_folder +phases: + install: + runtime-versions: + docker: 19 + build: + commands: + - | + if [[ $CODEBUILD_SOURCE_VERSION == "pr/"* ]]; then + pr_number=${CODEBUILD_SOURCE_VERSION#pr/} + api_response=$(curl -s "https://api.github.com/repos/shadow-robot/aurora/pulls/$pr_number") + if [[ $(echo $api_response | jq -r '.head.ref') == 'null' ]]; then + echo "$api_response" | jq + fi + branch=$(echo $api_response | jq -r '.head.ref') + elif [[ $CODEBUILD_SOURCE_VERSION =~ ^[0-9a-f]{40}$ ]]; then + branch=$(git ls-remote --symref origin | grep -m1 $CODEBUILD_SOURCE_VERSION | awk -F '/' '{print $NF}') + else + branch=$CODEBUILD_SOURCE_VERSION + fi + - cd .. + - rm -rf aurora + - git clone https://github.com/shadow-robot/aurora.git + - cd aurora + - git checkout $branch + - awscli_v1_executable=$(which aws) + - rm $awscli_v1_executable + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - cd $dockerfile_folder + - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot + - docker build --build-arg ml_docker_aurora_branch=$branch -t $image . + - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag + - docker push public.ecr.aws/shadowrobot/$image:$tag + - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt + - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi diff --git a/docker/molecule/docker/jammy/docker-compose.test.yml b/docker/molecule/docker/jammy/docker-compose.test.yml new file mode 100644 index 000000000..5aa3a844a --- /dev/null +++ b/docker/molecule/docker/jammy/docker-compose.test.yml @@ -0,0 +1,25 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: '3' + +services: + sut: + build: + context: . + args: + ml_docker_aurora_branch: ${SOURCE_BRANCH} + command: docker --version + environment: + - SOURCE_BRANCH diff --git a/docker/molecule/docker/jammy/hooks/build b/docker/molecule/docker/jammy/hooks/build new file mode 100755 index 000000000..a56600c93 --- /dev/null +++ b/docker/molecule/docker/jammy/hooks/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +export DOCKERFILE_PATH=${DOCKERFILE_PATH:-Dockerfile} + +echo "Executing build command" +echo "docker build --build-arg ml_docker_aurora_branch=${SOURCE_BRANCH} -f ${DOCKERFILE_PATH} -t ${IMAGE_NAME} ." + +docker build --build-arg ml_docker_aurora_branch="${SOURCE_BRANCH}" -f "${DOCKERFILE_PATH}" -t "${IMAGE_NAME}" . diff --git a/docker/molecule/ec2/jammy/Dockerfile b/docker/molecule/ec2/jammy/Dockerfile new file mode 100644 index 000000000..fad4c2294 --- /dev/null +++ b/docker/molecule/ec2/jammy/Dockerfile @@ -0,0 +1,32 @@ +FROM public.ecr.aws/shadowrobot/aurora-molecule:jammy + +LABEL Description="Ubuntu Jammy image contains Ansible, Molecule and EC2 driver" Vendor="Shadow Robot" Version="1.0" + +ARG ml_ec2_aurora_branch + +ENV ml_ec2_aurora_branch ${ml_ec2_aurora_branch:-"master"} + +ENV ml_ec2_aurora_script "https://raw.githubusercontent.com/shadow-robot/aurora/$ml_ec2_aurora_branch/bin/run-ansible.sh" + +RUN set -x && \ + \ + echo "Installing AWS CLI" && \ + wget -O /tmp/oneliner "$( echo "$ml_ec2_aurora_script" | sed 's/#/%23/g' )" && \ + chmod 755 /tmp/oneliner && \ + /tmp/oneliner install_software --branch $ml_ec2_aurora_branch software=[aws-cli] && \ + \ + wget https://s3.eu-west-2.amazonaws.com/amazon-ssm-eu-west-2/latest/debian_amd64/amazon-ssm-agent.deb && \ + dpkg -i amazon-ssm-agent.deb && \ + \ + echo "Clean up" && \ + apt-get clean && \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /root/.ansible \ + /root/.gitconfig \ + /root/.cache + +VOLUME /var/lib/docker + diff --git a/docker/molecule/ec2/jammy/README.md b/docker/molecule/ec2/jammy/README.md new file mode 100644 index 000000000..16625bc39 --- /dev/null +++ b/docker/molecule/ec2/jammy/README.md @@ -0,0 +1 @@ +Please read [these instructions](../../../../docs/molecule_dockers.md). diff --git a/docker/molecule/ec2/jammy/buildspec.yml b/docker/molecule/ec2/jammy/buildspec.yml new file mode 100644 index 000000000..8d483c7b0 --- /dev/null +++ b/docker/molecule/ec2/jammy/buildspec.yml @@ -0,0 +1,52 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: 0.2 + +env: + shell: bash + variables: + branch: master + image: aurora-molecule-ec2 + tag: jammy + dockerfile_folder: docker/molecule/ec2/jammy/ + exported-variables: + - branch + - image + - tag + - dockerfile_folder +phases: + install: + runtime-versions: + docker: 19 + build: + commands: + - cd .. + - rm -rf aurora + - git clone https://github.com/shadow-robot/aurora.git + - cd aurora + - git checkout $branch + - awscli_v1_executable=$(which aws) + - rm $awscli_v1_executable + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - cd $dockerfile_folder + - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot + - docker build -t $image . + - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag + - docker push public.ecr.aws/shadowrobot/$image:$tag + - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt + - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi diff --git a/docker/molecule/ec2/jammy/docker-compose.test.yml b/docker/molecule/ec2/jammy/docker-compose.test.yml new file mode 100644 index 000000000..f58338a43 --- /dev/null +++ b/docker/molecule/ec2/jammy/docker-compose.test.yml @@ -0,0 +1,25 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: '3' + +services: + sut: + build: + context: . + args: + ml_ec2_aurora_branch: ${SOURCE_BRANCH} + command: aws --version + environment: + - SOURCE_BRANCH diff --git a/docker/molecule/ec2/jammy/hooks/build b/docker/molecule/ec2/jammy/hooks/build new file mode 100755 index 000000000..97a434bf4 --- /dev/null +++ b/docker/molecule/ec2/jammy/hooks/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +export DOCKERFILE_PATH=${DOCKERFILE_PATH:-Dockerfile} + +echo "Executing build command" +echo "docker build --build-arg ml_ec2_aurora_branch=${SOURCE_BRANCH} -f ${DOCKERFILE_PATH} -t ${IMAGE_NAME} ." + +docker build --build-arg ml_ec2_aurora_branch="${SOURCE_BRANCH}" -f "${DOCKERFILE_PATH}" -t "${IMAGE_NAME}" . diff --git a/docker/test/docker/jammy/Dockerfile b/docker/test/docker/jammy/Dockerfile new file mode 100644 index 000000000..a2a0cdfc1 --- /dev/null +++ b/docker/test/docker/jammy/Dockerfile @@ -0,0 +1,38 @@ +FROM public.ecr.aws/shadowrobot/aurora-test-ubuntu:jammy + +LABEL Description="This image is used for testing Ubuntu Jammy with Docker deployment" Vendor="Shadow Robot" Version="1.0" + +ARG DEBIAN_FRONTEND=noninteractive + +ENV LC_ALL C.UTF-8 + +ENV LANG C.UTF-8 + +ENV USER testuser + +ARG ts_docker_aurora_branch + +ENV ts_docker_aurora_branch ${ts_docker_aurora_branch:-"master"} + +ENV ts_docker_aurora_script "https://raw.githubusercontent.com/shadow-robot/aurora/$ts_docker_aurora_branch/bin/run-ansible.sh" + +RUN set -x && \ + sudo apt-get update && \ + \ + echo "Installing wget" && \ + sudo apt-get install -y --no-install-recommends ca-certificates wget && \ + \ + echo "Installing Docker" && \ + wget -O /tmp/oneliner "$( echo "$ts_docker_aurora_script" | sed 's/#/%23/g' )" && \ + chmod 755 /tmp/oneliner && \ + /tmp/oneliner install_software --branch $ts_docker_aurora_branch software=[docker] skip_molecule_task=true && \ + \ + echo "Clean up" && \ + sudo apt-get clean && \ + sudo rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /root/.ansible \ + /root/.gitconfig \ + /root/.cache diff --git a/docker/test/docker/jammy/buildspec.yml b/docker/test/docker/jammy/buildspec.yml new file mode 100644 index 000000000..c4200aace --- /dev/null +++ b/docker/test/docker/jammy/buildspec.yml @@ -0,0 +1,63 @@ +# Copyright 2022-2023 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: 0.2 + +env: + shell: bash + variables: + image: aurora-test-ubuntu-docker + tag: jammy + dockerfile_folder: docker/test/docker/jammy/ + exported-variables: + - image + - tag + - dockerfile_folder +phases: + install: + runtime-versions: + docker: 19 + build: + commands: + - | + if [[ $CODEBUILD_SOURCE_VERSION == "pr/"* ]]; then + pr_number=${CODEBUILD_SOURCE_VERSION#pr/} + api_response=$(curl -s "https://api.github.com/repos/shadow-robot/aurora/pulls/$pr_number") + if [[ $(echo $api_response | jq -r '.head.ref') == 'null' ]]; then + echo "$api_response" | jq + fi + branch=$(echo $api_response | jq -r '.head.ref') + elif [[ $CODEBUILD_SOURCE_VERSION =~ ^[0-9a-f]{40}$ ]]; then + branch=$(git ls-remote --symref origin | grep -m1 $CODEBUILD_SOURCE_VERSION | awk -F '/' '{print $NF}') + else + branch=$CODEBUILD_SOURCE_VERSION + fi + - cd .. + - rm -rf aurora + - git clone https://github.com/shadow-robot/aurora.git + - cd aurora + - git checkout $branch + - awscli_v1_executable=$(which aws) + - rm $awscli_v1_executable + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - cd $dockerfile_folder + - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot + - docker build --build-arg ts_docker_aurora_branch=$branch -t $image . + - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag + - docker push public.ecr.aws/shadowrobot/$image:$tag + - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt + - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi diff --git a/docker/test/docker/jammy/docker-compose.test.yml b/docker/test/docker/jammy/docker-compose.test.yml new file mode 100644 index 000000000..1526cebf8 --- /dev/null +++ b/docker/test/docker/jammy/docker-compose.test.yml @@ -0,0 +1,25 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: '3' + +services: + sut: + build: + context: . + args: + ts_docker_aurora_branch: ${SOURCE_BRANCH} + command: docker --version + environment: + - SOURCE_BRANCH diff --git a/docker/test/docker/jammy/hooks/build b/docker/test/docker/jammy/hooks/build new file mode 100755 index 000000000..a132d3e08 --- /dev/null +++ b/docker/test/docker/jammy/hooks/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +export DOCKERFILE_PATH=${DOCKERFILE_PATH:-Dockerfile} + +echo "Executing build command" +echo "docker build --build-arg ts_docker_aurora_branch=${SOURCE_BRANCH} -f ${DOCKERFILE_PATH} -t ${IMAGE_NAME} ." + +docker build --build-arg ts_docker_aurora_branch="${SOURCE_BRANCH}" -f "${DOCKERFILE_PATH}" -t "${IMAGE_NAME}" . diff --git a/docker/test/ubuntu/jammy/Dockerfile b/docker/test/ubuntu/jammy/Dockerfile new file mode 100644 index 000000000..c76caa3e2 --- /dev/null +++ b/docker/test/ubuntu/jammy/Dockerfile @@ -0,0 +1,31 @@ +FROM public.ecr.aws/shadowrobot/ubuntu:jammy + +LABEL Description="This image is used for testing empty Ubuntu Jammy deployment" Vendor="Shadow Robot" Version="1.0" + +ENV DEBIAN_FRONTEND noninteractive + +ENV USER testuser + +RUN set -x && \ + apt-get update && \ + \ + echo "Installing sudo" && \ + apt-get install -y sudo && \ + \ + echo "Create testuser with empty password" && \ + useradd -m $USER && \ + passwd -d $USER && \ + adduser $USER sudo && \ + echo "$USER ALL=(ALL) NOPASSWD:ALL" | tee -a /etc/sudoers && \ + \ + echo "Clean up" && \ + apt-get clean && \ + rm -rf \ + /var/lib/apt/lists/* \ + /tmp/* \ + /var/tmp/* \ + /root/.ansible \ + /root/.gitconfig \ + /root/.cache + +USER $USER diff --git a/docker/test/ubuntu/jammy/buildspec.yml b/docker/test/ubuntu/jammy/buildspec.yml new file mode 100644 index 000000000..fc605d802 --- /dev/null +++ b/docker/test/ubuntu/jammy/buildspec.yml @@ -0,0 +1,52 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: 0.2 + +env: + shell: bash + variables: + branch: master + image: aurora-test-ubuntu + tag: jammy + dockerfile_folder: docker/test/ubuntu/jammy/ + exported-variables: + - branch + - image + - tag + - dockerfile_folder +phases: + install: + runtime-versions: + docker: 19 + build: + commands: + - cd .. + - rm -rf aurora + - git clone https://github.com/shadow-robot/aurora.git + - cd aurora + - git checkout $branch + - awscli_v1_executable=$(which aws) + - rm $awscli_v1_executable + - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + - unzip awscliv2.zip + - ./aws/install + - cd $dockerfile_folder + - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot + - docker build -t $image . + - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag + - docker push public.ecr.aws/shadowrobot/$image:$tag + - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt + - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi diff --git a/docker/test/ubuntu/jammy/docker-compose.test.yml b/docker/test/ubuntu/jammy/docker-compose.test.yml new file mode 100644 index 000000000..82244d3ef --- /dev/null +++ b/docker/test/ubuntu/jammy/docker-compose.test.yml @@ -0,0 +1,20 @@ +# Copyright 2022 Shadow Robot Company Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation version 2 of the License. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +version: '3' + +services: + sut: + build: . + command: id -u testuser From d972cb2ecda4c9693817fb239e388107d9dac2e9 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 13:04:17 +0100 Subject: [PATCH 04/11] docker runtime version not needed --- docker/test/ubuntu/jammy/buildspec.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/test/ubuntu/jammy/buildspec.yml b/docker/test/ubuntu/jammy/buildspec.yml index fc605d802..bdfd4f937 100644 --- a/docker/test/ubuntu/jammy/buildspec.yml +++ b/docker/test/ubuntu/jammy/buildspec.yml @@ -27,9 +27,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - cd .. From 8824fa854e094ccad64e8c8d706c6c4a20f6c4df Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 13:08:58 +0100 Subject: [PATCH 05/11] runtime version not needed, aws reinstall not needed --- docker/molecule/base/jammy/buildspec.yml | 8 -------- docker/molecule/devel/jammy/buildspec.yml | 8 -------- docker/molecule/docker/jammy/buildspec.yml | 8 -------- docker/molecule/ec2/jammy/buildspec.yml | 8 -------- docker/test/docker/jammy/buildspec.yml | 8 -------- docker/test/ubuntu/jammy/buildspec.yml | 5 ----- 6 files changed, 45 deletions(-) diff --git a/docker/molecule/base/jammy/buildspec.yml b/docker/molecule/base/jammy/buildspec.yml index a0ebcdf5b..7183e7d71 100644 --- a/docker/molecule/base/jammy/buildspec.yml +++ b/docker/molecule/base/jammy/buildspec.yml @@ -26,9 +26,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - cd .. @@ -36,11 +33,6 @@ phases: - git clone https://github.com/shadow-robot/aurora.git - cd aurora - git checkout $branch - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - docker build -t $image . diff --git a/docker/molecule/devel/jammy/buildspec.yml b/docker/molecule/devel/jammy/buildspec.yml index 0938a6b9c..a46b8cb28 100644 --- a/docker/molecule/devel/jammy/buildspec.yml +++ b/docker/molecule/devel/jammy/buildspec.yml @@ -27,9 +27,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - cd .. @@ -37,11 +34,6 @@ phases: - git clone https://github.com/shadow-robot/aurora.git - cd aurora - git checkout $branch - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - docker build -t $image . diff --git a/docker/molecule/docker/jammy/buildspec.yml b/docker/molecule/docker/jammy/buildspec.yml index 9c1dacf3d..21dd788b9 100644 --- a/docker/molecule/docker/jammy/buildspec.yml +++ b/docker/molecule/docker/jammy/buildspec.yml @@ -25,9 +25,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - | @@ -48,11 +45,6 @@ phases: - git clone https://github.com/shadow-robot/aurora.git - cd aurora - git checkout $branch - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - docker build --build-arg ml_docker_aurora_branch=$branch -t $image . diff --git a/docker/molecule/ec2/jammy/buildspec.yml b/docker/molecule/ec2/jammy/buildspec.yml index 8d483c7b0..f96a35e6d 100644 --- a/docker/molecule/ec2/jammy/buildspec.yml +++ b/docker/molecule/ec2/jammy/buildspec.yml @@ -27,9 +27,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - cd .. @@ -37,11 +34,6 @@ phases: - git clone https://github.com/shadow-robot/aurora.git - cd aurora - git checkout $branch - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - docker build -t $image . diff --git a/docker/test/docker/jammy/buildspec.yml b/docker/test/docker/jammy/buildspec.yml index c4200aace..893c22f57 100644 --- a/docker/test/docker/jammy/buildspec.yml +++ b/docker/test/docker/jammy/buildspec.yml @@ -25,9 +25,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - | @@ -48,11 +45,6 @@ phases: - git clone https://github.com/shadow-robot/aurora.git - cd aurora - git checkout $branch - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - docker build --build-arg ts_docker_aurora_branch=$branch -t $image . diff --git a/docker/test/ubuntu/jammy/buildspec.yml b/docker/test/ubuntu/jammy/buildspec.yml index bdfd4f937..0ca61a75d 100644 --- a/docker/test/ubuntu/jammy/buildspec.yml +++ b/docker/test/ubuntu/jammy/buildspec.yml @@ -34,11 +34,6 @@ phases: - git clone https://github.com/shadow-robot/aurora.git - cd aurora - git checkout $branch - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - docker build -t $image . From 7784a0bc09b88763ea353301da28aa60f10c7297 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 14:04:15 +0100 Subject: [PATCH 06/11] debugging --- docker/test/ubuntu/jammy/buildspec.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/test/ubuntu/jammy/buildspec.yml b/docker/test/ubuntu/jammy/buildspec.yml index 0ca61a75d..53e3acca3 100644 --- a/docker/test/ubuntu/jammy/buildspec.yml +++ b/docker/test/ubuntu/jammy/buildspec.yml @@ -41,4 +41,15 @@ phases: - docker push public.ecr.aws/shadowrobot/$image:$tag - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt - ids_to_delete=$(sed 's/^/imageDigest=/' ids_to_delete.txt) + - aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u >> t1.txt + - aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u >> t2.txt + - echo 't1' + - cat t1.txt + - echo 't2' + - cat t2.txt + - echo 'ids1' + - echo $ids_to_delete + - echo 'ids2txt' + - echo 'img' + - echo $image - if [[ -s ids_to_delete.txt ]]; then aws ecr-public batch-delete-image --repository-name $image --region us-east-1 --image-ids $ids_to_delete --output text; fi From 7bd0ece1257b0e6f7c918d3a01d2041a47d4df85 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 15:43:41 +0100 Subject: [PATCH 07/11] cython>3 breaks pyyaml install --- ansible/data/molecule/constraint.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 ansible/data/molecule/constraint.txt diff --git a/ansible/data/molecule/constraint.txt b/ansible/data/molecule/constraint.txt new file mode 100644 index 000000000..039eb0db2 --- /dev/null +++ b/ansible/data/molecule/constraint.txt @@ -0,0 +1 @@ +cython<3 From 64928350e4c07b917f18bf5c697b6acac23ed2ef Mon Sep 17 00:00:00 2001 From: Tom Queen Date: Thu, 19 Sep 2024 16:19:42 +0100 Subject: [PATCH 08/11] Update README.md From 69347311a0be3ae92c143c189b9daab961d544ed Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 16:50:10 +0100 Subject: [PATCH 09/11] fix for jammy --- docker/molecule/base/jammy/Dockerfile | 10 +++++----- docker/molecule/base/jammy/buildspec.yml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/molecule/base/jammy/Dockerfile b/docker/molecule/base/jammy/Dockerfile index dc9ea2aa6..71b8855ab 100644 --- a/docker/molecule/base/jammy/Dockerfile +++ b/docker/molecule/base/jammy/Dockerfile @@ -23,7 +23,7 @@ RUN set -x && \ apt-get install -y --no-install-recommends ca-certificates wget && \ \ echo "Installing dependencies for python3.8, SSL, pip3" && \ - apt-get install -y build-essential zlib1g-dev libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \ + apt-get install -y build-essential zlib1g-dev libreadline-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev && \ \ echo "Installing git, sudo and Python 3 pip and dev" && \ apt-get install -y \ @@ -31,20 +31,20 @@ RUN set -x && \ python3-pip \ git \ libyaml-dev \ - python-crypto \ + python3-cryptography \ libssl-dev \ libffi-dev \ sshpass \ sudo && \ - \ - pip3 install --upgrade pip setuptools==51.1.1 gcovr && \ + pip3 install --upgrade pip gcovr && \ echo "Cloning Aurora" && \ git clone --depth 1 https://github.com/shadow-robot/aurora.git -b $ml_base_aurora_branch /tmp/aurora && \ \ pip3 --version && \ python3 --version && \ echo "Installing molecule" && \ - pip3 install -r /tmp/aurora/ansible/data/molecule/requirements.txt && \ + echo "cython<3" > /tmp/constraint.txt && \ + PIP_CONSTRAINT=/tmp/constraint.txt pip3 install -r /tmp/aurora/ansible/data/molecule/requirements.txt && \ \ echo "Clean up" && \ apt-get clean && \ diff --git a/docker/molecule/base/jammy/buildspec.yml b/docker/molecule/base/jammy/buildspec.yml index 7183e7d71..5b6e3321c 100644 --- a/docker/molecule/base/jammy/buildspec.yml +++ b/docker/molecule/base/jammy/buildspec.yml @@ -18,8 +18,8 @@ env: variables: branch: master image: aurora-molecule - tag: focal - dockerfile_folder: docker/molecule/base/focal/ + tag: jammy + dockerfile_folder: docker/molecule/base/jammy/ exported-variables: - branch - image From 7bd65984e1f7c733490c3c66699cb14f737e8281 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 16:50:41 +0100 Subject: [PATCH 10/11] upgrade ec2 ami to use ubuntu 20.04 --- .../server_and_nuc_desktop_icons_docker_ec2/molecule.yml | 2 +- .../molecule.yml | 2 +- .../teleop_server_check_desktop_icons_docker_ec2/molecule.yml | 2 +- .../teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml index dc8fe77b6..80e5f4d8b 100644 --- a/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: server_and_nuc_desktop_icons_docker_ec2_${CODEBUILD_BUILD_ID} - image: ami-0820357ff5cf2333d + image: ami-05926bb0a947f9e7f instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 diff --git a/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml index b5e7a8abd..352e7ad20 100644 --- a/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: teleop_server_desktop_icons_simulation_haptx_ec2_${CODEBUILD_BUILD_ID} - image: ami-0820357ff5cf2333d + image: ami-05926bb0a947f9e7f instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 diff --git a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml index b36e397b6..bd8c439cf 100644 --- a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: teleop_server_check_desktop_icons_docker_ec2_${CODEBUILD_BUILD_ID} - image: ami-0820357ff5cf2333d + image: ami-05926bb0a947f9e7f instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 diff --git a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml index 8d5d37a5b..221894072 100644 --- a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: teleop_server_desktop_icons_haptx_bimanual_ec2_${CODEBUILD_BUILD_ID} - image: ami-0820357ff5cf2333d + image: ami-05926bb0a947f9e7f instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 From 4ab76b1246b2314ac40226eb8d0522c3a4057c12 Mon Sep 17 00:00:00 2001 From: Build Tools Date: Thu, 19 Sep 2024 17:05:07 +0100 Subject: [PATCH 11/11] accidentally used arm64 ami, move to amd64 --- .../server_and_nuc_desktop_icons_docker_ec2/molecule.yml | 2 +- .../molecule.yml | 2 +- .../teleop_server_check_desktop_icons_docker_ec2/molecule.yml | 2 +- .../teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml index 80e5f4d8b..ae59fda1c 100644 --- a/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_server_and_nuc/molecule/server_and_nuc_desktop_icons_docker_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: server_and_nuc_desktop_icons_docker_ec2_${CODEBUILD_BUILD_ID} - image: ami-05926bb0a947f9e7f + image: ami-00826bd51e68b1487 instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 diff --git a/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml index 352e7ad20..6d411b927 100644 --- a/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_simulation/molecule/teleop_server_desktop_icons_simulation_haptx_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: teleop_server_desktop_icons_simulation_haptx_ec2_${CODEBUILD_BUILD_ID} - image: ami-05926bb0a947f9e7f + image: ami-00826bd51e68b1487 instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 diff --git a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml index bd8c439cf..234e049d6 100644 --- a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_check_desktop_icons_docker_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: teleop_server_check_desktop_icons_docker_ec2_${CODEBUILD_BUILD_ID} - image: ami-05926bb0a947f9e7f + image: ami-00826bd51e68b1487 instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4 diff --git a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml index 221894072..041d5d8a7 100644 --- a/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml +++ b/ansible/playbooks/molecule_ec2_teleop/molecule/teleop_server_desktop_icons_haptx_bimanual_ec2/molecule.yml @@ -26,7 +26,7 @@ lint: | platforms: # Adding CODEBUILD_BUILD_ID to instance name in order to allow parallel EC2 execution of tests from CodeBuild - name: teleop_server_desktop_icons_haptx_bimanual_ec2_${CODEBUILD_BUILD_ID} - image: ami-05926bb0a947f9e7f + image: ami-00826bd51e68b1487 instance_type: t2.micro region: eu-west-2 vpc_id: vpc-0f8cc2cc245d57eb4