[tests] build a local gantry image to run container tests. Also used …

…it as images remover.
shizunge · Nov 28, 2024 · 042f736 · 042f736
1 parent b55af24
commit 042f736
Show file tree

Hide file tree

Showing 21 changed files with 276 additions and 204 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -36,8 +36,6 @@ jobs:
         uses: actions/checkout@v4
       - name: Generate tests coverage reports
         run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
           bash shellspec --kcov --jobs 50
       - name: Upload reports
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml
@@ -66,3 +66,26 @@ jobs:
         run: |
           bash shellspec --pattern tests/${{ matrix.test_suit }}
 
+  container_tests:
+    name: Run container tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+      - name: Install shellspec
+        run: |
+          mkdir -p ~/shellspec
+          cd ~/shellspec
+          git clone https://github.com/shellspec/shellspec.git
+          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
+          echo -n "shellspec version: "
+          shellspec --version
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Run container tests
+        run: |
+          export GANTRY_TEST_CONTAINER=true
+          export GANTRY_TEST_CONTAINER_REPO_TAG=
+          echo "GANTRY_TEST_CONTAINER=${GANTRY_TEST_CONTAINER}"
+          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
+          bash shellspec --jobs 50
diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml
@@ -67,17 +67,40 @@ jobs:
         uses: actions/checkout@v4
       - name: Run tests
         run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
           bash shellspec --pattern tests/${{ matrix.test_suit }}
 
+  container_tests:
+    name: Run container tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+      - name: Install shellspec
+        run: |
+          mkdir -p ~/shellspec
+          cd ~/shellspec
+          git clone https://github.com/shellspec/shellspec.git
+          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
+          echo -n "shellspec version: "
+          shellspec --version
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Run container tests
+        run: |
+          export GANTRY_TEST_CONTAINER=true
+          export GANTRY_TEST_CONTAINER_REPO_TAG=
+          echo "GANTRY_TEST_CONTAINER=${GANTRY_TEST_CONTAINER}"
+          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
+          bash shellspec --jobs 50
+
   build_and_push:
     name: Build and push Docker image
     runs-on: ubuntu-latest
     if: ${{ github.actor != 'dependabot[bot]' }}
     needs:
       - shellcheck
       - tests
+      - container_tests
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -116,50 +139,3 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           provenance: false
-      - name: Extract tag
-        run: |
-          TAGS="${{ steps.meta.outputs.tags }}"
-          for TAG in ${TAGS}; do
-            if echo "${TAG}" | grep -q "ghcr.io/${{ github.repository }}-development:dev-"; then
-              echo "TAG=${TAG}"
-              echo "${TAG}" > tag.txt
-              break;   
-            fi
-          done
-      - name: Store tag
-        uses: actions/upload-artifact@v4
-        with:
-          name: tag
-          path: tag.txt
-
-
-  container_tests:
-    name: Test container
-    runs-on: ubuntu-latest
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    needs:
-      - build_and_push
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/[email protected]
-      - name: Install shellspec
-        run: |
-          mkdir -p ~/shellspec
-          cd ~/shellspec
-          git clone https://github.com/shellspec/shellspec.git
-          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
-          echo -n "shellspec version: "
-          shellspec --version
-      - name: Checkout Code
-        uses: actions/checkout@v4
-      - name: Load tag
-        uses: actions/download-artifact@v4
-        with:
-          name: tag
-      - name: Run tests
-        run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
-          export GANTRY_TEST_CONTAINER_REPO_TAG=$(cat tag.txt)
-          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
-          bash shellspec --jobs 50
diff --git a/.github/workflows/on-release.yml b/.github/workflows/on-release.yml
@@ -42,8 +42,30 @@ jobs:
         uses: actions/checkout@v4
       - name: Run tests
         run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
+          bash shellspec --jobs 50
+
+  container_tests:
+    name: Run container tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+      - name: Install shellspec
+        run: |
+          mkdir -p ~/shellspec
+          cd ~/shellspec
+          git clone https://github.com/shellspec/shellspec.git
+          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
+          echo -n "shellspec version: "
+          shellspec --version
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Run container tests
+        run: |
+          export GANTRY_TEST_CONTAINER=true
+          export GANTRY_TEST_CONTAINER_REPO_TAG=
+          echo "GANTRY_TEST_CONTAINER=${GANTRY_TEST_CONTAINER}"
+          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
           bash shellspec --jobs 50
 
   build_and_push:
@@ -52,6 +74,7 @@ jobs:
     needs:
       - shellcheck
       - tests
+      - container_tests
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4

diff --git a/tests/README.md b/tests/README.md
@@ -32,7 +32,17 @@ To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) inst
 bash shellspec --kcov
 ```
 
-If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
+The above commands test *Gantry* as a script running on the host directly. We also want to test *Gantry* running inside a container in case the environments are different between the host and the container.
+
+To test *Gantry* running inside a container, set the environment variable `GANTRY_TEST_CONTAINER` to `true`. The testing framework will build a local image of *Gantry*, then start a service of that image to run the test.
+
+```
+export GANTRY_TEST_CONTAINER=true
+bash shellspec --jobs 50
+```
+
+If you want to test a specific image of *Gantry*, you need to set the image of *Gantry* explicitly via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
+
 ```
 export GANTRY_TEST_CONTAINER_REPO_TAG=<gantry image>:<tag>
 bash shellspec --jobs 50

diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
@@ -27,9 +27,9 @@ Describe 'cleanup-images'
     test_CLEANUP_IMAGES_false() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_CLEANUP_IMAGES="false"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -70,11 +70,11 @@ Describe 'cleanup-images'
     test_CLEANUP_IMAGES_OPTIONS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_CLEANUP_IMAGES="true"
       # Image remover would fail due to the incorrect option.
       export GANTRY_CLEANUP_IMAGES_OPTIONS="--incorrect-option"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -117,10 +117,10 @@ Describe 'cleanup-images'
     test_CLEANUP_IMAGES_OPTIONS_good() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_CLEANUP_IMAGES="true"
       export GANTRY_CLEANUP_IMAGES_OPTIONS="--container-label=test"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -194,9 +194,9 @@ Describe 'cleanup-images'
       local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0"
       local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
       local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_IMAGES_TO_REMOVE="${IMAGE_WITH_TAG0} ${IMAGE_WITH_TAG1} ${IMAGE_WITH_TAG2}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     test_end() {
       local TEST_NAME="${1}"

diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
@@ -26,9 +26,9 @@ Describe 'common-options'
     test_common_DOCKER_HOST_not_swarm_manager() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_HOST="8.8.8.8:53"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -72,10 +72,10 @@ Describe 'common-options'
     test_common_LOG_LEVEL_none() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Same as test_new_image_yes, except set LOG_LEVEL to NONE
       export GANTRY_LOG_LEVEL=NONE
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -106,11 +106,11 @@ Describe 'common-options'
       local ENV_AFTER_RUN=
       ENV_AFTER_RUN=$(mktemp)
 
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # There should be no warnings or errors. So it should work the same as LOG_LEVLE=NONE.
       export GANTRY_LOG_LEVEL=WARN
       declare -p > "${ENV_BEFORE_RUN}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       declare -p > "${ENV_AFTER_RUN}"
       # Allow the 3 mismatches LOG_LEVEL NODE_NAME LOG_SCOPE used in log() function.
       # Allow the 2 mismatches LINENO _ for kcov coverage.
@@ -140,7 +140,7 @@ Describe 'common-options'
     test_common_PRE_POST_RUN_CMD() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_OPTIONS=
       export GANTRY_CLEANUP_IMAGES=
       # Test that pre-run command can change the global configurations.
@@ -149,7 +149,7 @@ Describe 'common-options'
       local POST_CMD="for I in \$(seq 3 5); do echo \"TEST_OUTPUT_MULTIPLE_LINES=\$I\"; done"
       # Test that the command returns a non-zero value.
       export GANTRY_POST_RUN_CMD="echo \"Post update\"; ${POST_CMD}; false;"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -201,10 +201,10 @@ Describe 'common-options'
     test_common_SLEEP_SECONDS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SLEEP_SECONDS="7"
       # Run run_gantry in background.
-      run_gantry "${TEST_NAME}" &
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}" &
       local PID="${!}"
       sleep $((GANTRY_SLEEP_SECONDS*3+1))
       stop_gantry_container "${TEST_NAME}"
@@ -254,9 +254,9 @@ Describe 'common-options'
     test_common_SLEEP_SECONDS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SLEEP_SECONDS="NotANumber"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"

diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
@@ -26,9 +26,9 @@ Describe 'filters'
     test_SERVICES_FILTERS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SERVICES_FILTERS="BadFilterValue"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -75,7 +75,7 @@ Describe 'filters'
       local MAX_SERVICES_NUM="${3}"
       local NUM_SERVICES_EXCLUDED=$((MAX_SERVICES_NUM/2))
       local NUM_SERVICES_EXCLUDED_FILTER_START=$((NUM_SERVICES_EXCLUDED+1))
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       for NUM in $(seq 0 "${NUM_SERVICES_EXCLUDED}"); do
         local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
         export GANTRY_SERVICES_EXCLUDED="${GANTRY_SERVICES_EXCLUDED} ${SERVICE_NAME_NUM}"
@@ -86,7 +86,7 @@ Describe 'filters'
         docker_service_update --label-add "${LABEL}=true" "${SERVICE_NAME_NUM}"
       done
       export GANTRY_SERVICES_EXCLUDED_FILTERS="label=${LABEL}=true"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
     AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
@@ -131,14 +131,14 @@ Describe 'filters'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local MAX_SERVICES_NUM="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       local LABEL="gantry.services.excluded"
       for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do
         local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
         docker_service_update --label-add "${LABEL}=true" "${SERVICE_NAME_NUM}"
       done
       # Do not set GANTRY_SERVICES_EXCLUDED_FILTERS, check the default one is working.
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
     AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
@@ -180,9 +180,9 @@ Describe 'filters'
     test_SERVICES_EXCLUDED_FILTERS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SERVICES_EXCLUDED_FILTERS="BadFilterValue"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"