[tests] allow more tests to run with a container.

shizunge · Nov 25, 2024 · 26ac05f · 26ac05f
1 parent d6bf10a
commit 26ac05f
Show file tree

Hide file tree

Showing 11 changed files with 141 additions and 104 deletions.
diff --git a/src/lib-common.sh b/src/lib-common.sh
@@ -557,7 +557,7 @@ _all_tasks_reach_state() {
   return 0
 }
 
-# Usage: wait_service_state <SERVICE_NAME> <WANT_STATE>
+# Usage: wait_service_state <SERVICE_NAME> <WANT_STATE> [timeout in seconds]
 # Wait for the service, usually a global job or a replicated job,
 # to reach either running or complete state.
 # Valid WANT_STATE includes "Running" and "Complete"
@@ -567,15 +567,26 @@ _all_tasks_reach_state() {
 wait_service_state() {
   local SERVICE_NAME="${1}";
   local WANT_STATE="${2}";
+  local TIMEOUT_SECONDS="${3}";
   local CHECK_FAILURES=false
   [ "${WANT_STATE}" = "Complete" ] && CHECK_FAILURES=true
   local SLEEP_SECONDS=1
+  local START_TIME=
+  START_TIME=$(date +%s)
   local DOCKER_CMD_ERROR=1
   local RETURN_VALUE=0
   local STATES=
   while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do
     DOCKER_CMD_ERROR=0
     RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break
+    local SECONDS_ELAPSED=
+    if is_number "${TIMEOUT_SECONDS}" \
+        && SECONDS_ELAPSED=$(first_minus_second "$(date +%s)" "${START_TIME}") \
+        && [ "${SECONDS_ELAPSED}" -ge "${TIMEOUT_SECONDS}" ]; then
+      log ERROR "wait_service_state ${SERVICE_NAME} timout after ${SECONDS_ELAPSED}s."
+      RETURN_VALUE=2
+      break
+    fi
     sleep "${SLEEP_SECONDS}"
     DOCKER_CMD_ERROR=1
   done

diff --git a/tests/README.md b/tests/README.md
@@ -29,14 +29,13 @@ bash shellspec --jobs 50
 
 To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) installed):
 ```
-bash shellspec --kcov --tag coverage:true
+bash shellspec --kcov --tag "coverage:true"
 ```
 
 If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
 ```
 export GANTRY_TEST_CONTAINER_REPO_TAG=<gantry image>:<tag>
-bash shellspec --tag "container_test:true" "coverage:true"
+bash shellspec --tag "container_test:true"
 ```
 
-> NOTE: Negative tests will hang when testing a *Gantry* container, which may be due to a bug in shellspec. So when testing *Gantry* images, we should run only tests with tag `container_test:true`.
-
+> NOTE: Some tests will fail when testing a *Gantry* container. So when testing *Gantry* images, we should run only tests with tag `container_test:true`.
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
@@ -19,7 +19,7 @@ Describe 'filters'
   SUITE_NAME="filters"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
+  Describe "test_SERVICES_FILTERS_bad" "container_test:true" "coverage:true"
     TEST_NAME="test_SERVICES_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -173,7 +173,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
+  Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:true" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")

diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
@@ -39,7 +39,10 @@ Describe 'login_docker_config'
     TEST_NAME="test_login_docker_config_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_no_label() {
       local TEST_NAME="${1}"
@@ -68,14 +71,14 @@ Describe 'login_docker_config'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -110,7 +113,10 @@ Describe 'login_docker_config'
     TEST_NAME="test_login_docker_config_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_default_config() {
       local TEST_NAME="${1}"
@@ -141,7 +147,7 @@ Describe 'login_docker_config'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -177,14 +183,17 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
+  Describe "test_login_docker_config_label_override" "container_test:true" "coverage:true"
     TEST_NAME="test_login_docker_config_label_override"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME0="${SERVICE_NAME}-0"
     SERVICE_NAME1="${SERVICE_NAME}-1"
     SERVICE_NAME2="${SERVICE_NAME}-2"
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_start() {
       local TEST_NAME="${1}"
@@ -241,21 +250,21 @@ Describe 'login_docker_config'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME0}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME2}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME2}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"