From 90f086598533ef98784e2bdd18bd91f9fec2ce0f Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Sun, 24 Nov 2024 22:32:19 -0800 Subject: [PATCH] [tests] allow more tests to run with a container. --- src/lib-common.sh | 27 ++++-- tests/README.md | 7 +- tests/gantry_common_options_spec.sh | 21 +++-- tests/gantry_filters_spec.sh | 4 +- tests/gantry_login_docker_config_spec.sh | 35 +++++--- tests/gantry_login_negative_spec.sh | 100 ++++++++++++++--------- tests/gantry_login_spec.sh | 30 ++++--- tests/gantry_manifest_spec.sh | 4 +- tests/gantry_notify_spec.sh | 2 +- tests/gantry_parallel_spec.sh | 4 +- tests/gantry_update_options_spec.sh | 2 +- tests/spec_gantry_test_helper.sh | 85 +++++++++++-------- 12 files changed, 194 insertions(+), 127 deletions(-) diff --git a/src/lib-common.sh b/src/lib-common.sh index c01ee1f..272ee16 100755 --- a/src/lib-common.sh +++ b/src/lib-common.sh @@ -21,11 +21,12 @@ _random_string() { _pipe_name() { local BASE_NAME="${1:-pipe-base-name}" - local RANDOM_STR= - RANDOM_STR=$(_random_string) + local PID=$$ local TIMESTAMP= TIMESTAMP=$(date +%s) - local PIPE_NAME="/tmp/${BASE_NAME}-$$-${TIMESTAMP}-${RANDOM_STR}" + local RANDOM_STR= + RANDOM_STR=$(_random_string) + local PIPE_NAME="/tmp/${BASE_NAME}-${PID}-${TIMESTAMP}-${RANDOM_STR}" echo "${PIPE_NAME}" } @@ -557,25 +558,35 @@ _all_tasks_reach_state() { return 0 } -# Usage: wait_service_state +# Usage: wait_service_state [timeout in seconds] # Wait for the service, usually a global job or a replicated job, # to reach either running or complete state. # Valid WANT_STATE includes "Running" and "Complete" -# When the WANT_STATE is complete, the function returns immediately -# when any of the tasks of the service fails. -# In case of task failing, the function returns a non-zero value. +# When the WANT_STATE is complete, the function returns immediately when any of the tasks of the service fails. +# In case of task failing, the function returns the first failing task's return value. wait_service_state() { local SERVICE_NAME="${1}"; local WANT_STATE="${2}"; + local TIMEOUT_SECONDS="${3}"; local CHECK_FAILURES=false [ "${WANT_STATE}" = "Complete" ] && CHECK_FAILURES=true local SLEEP_SECONDS=1 - local DOCKER_CMD_ERROR=1 + local START_TIME= + START_TIME=$(date +%s) local RETURN_VALUE=0 + local DOCKER_CMD_ERROR=1 local STATES= while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do DOCKER_CMD_ERROR=0 RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break + local SECONDS_ELAPSED= + if is_number "${TIMEOUT_SECONDS}" \ + && SECONDS_ELAPSED=$(first_minus_second "$(date +%s)" "${START_TIME}") \ + && [ "${SECONDS_ELAPSED}" -ge "${TIMEOUT_SECONDS}" ]; then + log ERROR "wait_service_state ${SERVICE_NAME} ${WANT_STATE} timeout after ${SECONDS_ELAPSED}s." + RETURN_VALUE=2 + break + fi sleep "${SLEEP_SECONDS}" DOCKER_CMD_ERROR=1 done diff --git a/tests/README.md b/tests/README.md index b334374..bbc54e7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -29,14 +29,13 @@ bash shellspec --jobs 50 To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) installed): ``` -bash shellspec --kcov --tag coverage:true +bash shellspec --kcov --tag "coverage:true" ``` If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`. ``` export GANTRY_TEST_CONTAINER_REPO_TAG=: -bash shellspec --tag "container_test:true" "coverage:true" +bash shellspec --tag "container_test:true" ``` -> NOTE: Negative tests will hang when testing a *Gantry* container, which may be due to a bug in shellspec. So when testing *Gantry* images, we should run only tests with tag `container_test:true`. - +> NOTE: Some tests will fail when testing a *Gantry* container. So when testing *Gantry* images, we should run only tests with tag `container_test:true`. diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh index f659d7d..e66f47a 100644 --- a/tests/gantry_common_options_spec.sh +++ b/tests/gantry_common_options_spec.sh @@ -19,7 +19,7 @@ Describe 'common-options' SUITE_NAME="common-options" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true" + Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:true" "coverage:true" TEST_NAME="test_common_DOCKER_HOST_not_swarm_manager" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") @@ -27,11 +27,11 @@ Describe 'common-options' local TEST_NAME="${1}" local SERVICE_NAME="${2}" reset_gantry_env "${SERVICE_NAME}" - export DOCKER_HOST="8.8.8.8:53" + export GANTRY_TEST_DOCKER_HOST="8.8.8.8:53" local RETURN_VALUE=0 run_gantry "${TEST_NAME}" RETURN_VALUE="${?}" - export DOCKER_HOST= + export GANTRY_TEST_DOCKER_HOST= return "${RETURN_VALUE}" } BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" @@ -93,10 +93,12 @@ Describe 'common-options' End End # Do not run test_common_no_new_env with the kcov, which alters the environment variables. - Describe "test_common_no_new_env" "container_test:false" "coverage:false" + Describe "test_common_no_new_env" "container_test:true" "coverage:false" # Check there is no new variable set, # to avoid errors like https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085 - # We don't need to run this test using containers because we check env on the host, while the container test set env inside the container. + # + # It makes no sense to run run this test using containers because we check env on the host, while the container test set env inside the container. + # But it should not failed with a container. We are just testing GANTRY_LOG_LEVEL=WARN. TEST_NAME="test_common_no_new_env" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") @@ -195,10 +197,7 @@ Describe 'common-options' The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}" End End - # run_gantry prints logs after gantry exists, while testing a container. - # In thes test, gantry never exit, but will be killed, thus there is no log. - # Therefore we disable the container test for this test. - Describe "test_common_SLEEP_SECONDS" "container_test:false" "coverage:true" + Describe "test_common_SLEEP_SECONDS" "container_test:true" "coverage:true" TEST_NAME="test_common_SLEEP_SECONDS" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") @@ -210,7 +209,7 @@ Describe 'common-options' run_gantry "${TEST_NAME}" & local PID="${!}" sleep $((GANTRY_SLEEP_SECONDS*3+1)) - kill "${PID}" + stop_gantry "${TEST_NAME}" "${PID}" } BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" @@ -249,7 +248,7 @@ Describe 'common-options' The stderr should satisfy spec_expect_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}" End End - Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false" "coverage:true" + Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:true" "coverage:true" TEST_NAME="test_common_SLEEP_SECONDS_not_a_number" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh index d14ebcb..7006474 100644 --- a/tests/gantry_filters_spec.sh +++ b/tests/gantry_filters_spec.sh @@ -19,7 +19,7 @@ Describe 'filters' SUITE_NAME="filters" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true" + Describe "test_SERVICES_FILTERS_bad" "container_test:true" "coverage:true" TEST_NAME="test_SERVICES_FILTERS_bad" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") @@ -173,7 +173,7 @@ Describe 'filters' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true" + Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:true" "coverage:true" TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh index d8a14db..c5bf351 100644 --- a/tests/gantry_login_docker_config_spec.sh +++ b/tests/gantry_login_docker_config_spec.sh @@ -39,7 +39,10 @@ Describe 'login_docker_config' TEST_NAME="test_login_docker_config_no_label" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_docker_config_no_label() { local TEST_NAME="${1}" @@ -68,14 +71,14 @@ Describe 'login_docker_config' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" @@ -110,7 +113,10 @@ Describe 'login_docker_config' TEST_NAME="test_login_docker_config_default_config" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_docker_config_default_config() { local TEST_NAME="${1}" @@ -141,7 +147,7 @@ Describe 'login_docker_config' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" @@ -177,14 +183,17 @@ Describe 'login_docker_config' The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true" + Describe "test_login_docker_config_label_override" "container_test:true" "coverage:true" TEST_NAME="test_login_docker_config_label_override" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") SERVICE_NAME0="${SERVICE_NAME}-0" SERVICE_NAME1="${SERVICE_NAME}-1" SERVICE_NAME2="${SERVICE_NAME}-2" - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_start() { local TEST_NAME="${1}" @@ -241,21 +250,21 @@ Describe 'login_docker_config' BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" - The stderr should satisfy spec_expect_message "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}" + The stderr should satisfy spec_expect_message "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}" # Check warnings The stderr should satisfy spec_expect_message "${THERE_ARE_NUM_CONFIGURATIONS}.*" The stderr should satisfy spec_expect_message "${USER_LOGGED_INTO_DEFAULT}.*" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME0}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}" - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME2}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME0}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME2}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh index 6db85d2..dfbd061 100644 --- a/tests/gantry_login_negative_spec.sh +++ b/tests/gantry_login_negative_spec.sh @@ -19,11 +19,14 @@ Describe 'login_negative' SUITE_NAME="login_negative" BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN" AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN" - Describe "test_login_no_login" "container_test:false" "coverage:true" + Describe "test_login_no_login" "container_test:true" "coverage:true" TEST_NAME="test_login_no_login" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_no_login() { local TEST_NAME="${1}" @@ -34,7 +37,7 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_no_login "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_no_login "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" @@ -70,11 +73,14 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_incorrect_password" "container_test:false" "coverage:true" + Describe "test_login_incorrect_password" "container_test:true" "coverage:true" TEST_NAME="test_login_incorrect_password" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_incorrect_password() { local TEST_NAME="${1}" @@ -104,14 +110,14 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_incorrect_password "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_incorrect_password "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}" - The stderr should satisfy spec_expect_message "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" @@ -143,7 +149,7 @@ Describe 'login_negative' TEST_NAME="test_login_read_only_file" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + AUTH_CONFIG=$(mktemp -d) TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_read_only_file() { local TEST_NAME="${1}" @@ -153,7 +159,8 @@ Describe 'login_negative' local USERNAME="${5}" local PASSWORD="${6}" check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1; - # Set the config folder to read only. (It won't work for container_test) + # When running with an image, we are not changing the folder inside the contianer. + # So do not run the test with a container/image. mkdir -p "${CONFIG}" chmod 444 "${CONFIG}" local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}"; @@ -175,14 +182,14 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_read_only_file "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_read_only_file "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}" - The stderr should satisfy spec_expect_message "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" @@ -210,11 +217,14 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_config_mismatch_default" "container_test:false" "coverage:true" + Describe "test_login_config_mismatch_default" "container_test:true" "coverage:true" TEST_NAME="test_login_config_mismatch_default" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_config_mismatch_default() { local TEST_NAME="${1}" @@ -254,21 +264,21 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" - The stderr should satisfy spec_expect_message "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}" + The stderr should satisfy spec_expect_message "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}" # Check warnings The stderr should satisfy spec_expect_message "${THERE_ARE_NUM_CONFIGURATIONS}.*" The stderr should satisfy spec_expect_message "${USER_LOGGED_INTO_DEFAULT}.*" - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" @@ -295,11 +305,14 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_config_mismatch_no_default" "container_test:false" "coverage:true" + Describe "test_login_config_mismatch_no_default" "container_test:true" "coverage:true" TEST_NAME="test_login_config_mismatch_no_default" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_config_mismatch_no_default() { local TEST_NAME="${1}" @@ -332,22 +345,22 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" - The stderr should satisfy spec_expect_message "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}" + The stderr should satisfy spec_expect_message "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}" # Check warnings The stderr should satisfy spec_expect_message "${THERE_ARE_NUM_CONFIGURATIONS}.*" # This message does not present, because we don't login with the default configuration. The stderr should satisfy spec_expect_no_message "${USER_LOGGED_INTO_DEFAULT}.*" - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" @@ -374,7 +387,7 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_multi_services_no_label" "container_test:false" "coverage:true" + Describe "test_login_multi_services_no_label" "container_test:true" "coverage:true" # To test https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085 TEST_NAME="test_login_multi_services_no_label" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") @@ -383,7 +396,10 @@ Describe 'login_negative' IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1" SERVICE_NAME0="${SERVICE_NAME}-0" SERVICE_NAME1="${SERVICE_NAME}-1" - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_start() { local TEST_NAME="${1}" @@ -446,16 +462,16 @@ Describe 'login_negative' BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_multi_services_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_multi_services_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" - The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME0}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}" + The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME0}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}" The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*" The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}" The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}" @@ -489,11 +505,14 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" "coverage:true" + Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:true" "coverage:true" TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_REGISTRY_CONFIGS_FILE_bad_format() { local TEST_NAME="${1}" @@ -522,7 +541,7 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" @@ -530,7 +549,7 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_message "format error.*Found extra item\(s\)" The stderr should satisfy spec_expect_message "format error.*Missing item\(s\)" - The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" @@ -559,11 +578,14 @@ Describe 'login_negative' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_login_file_not_exist" "container_test:false" "coverage:true" + Describe "test_login_file_not_exist" "container_test:true" "coverage:true" TEST_NAME="test_login_file_not_exist" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_file_not_exist() { local TEST_NAME="${1}" @@ -590,13 +612,13 @@ Describe 'login_negative' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be failure The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" - The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_message "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}" diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index 4c26906..ed3033b 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -26,7 +26,10 @@ Describe 'login' TEST_NAME="test_login_config" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_config() { local TEST_NAME="${1}" @@ -58,14 +61,14 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" @@ -74,7 +77,7 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME}" # 2 "--with-registry-auth" for SERVICE_NAME. One is automatically added. The other is from user. The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*automatically.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*specified by user.*${SERVICE_NAME}" @@ -100,7 +103,7 @@ Describe 'login' TEST_NAME="test_login_default_config" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="NotUsed" + AUTH_CONFIG="NotUsed" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_default_config() { local TEST_NAME="${1}" @@ -130,7 +133,7 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" @@ -170,7 +173,10 @@ Describe 'login' TEST_NAME="test_login_REGISTRY_CONFIGS_FILE" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="C$(unique_id)" + # When running with an Gantry image, docker buildx writes files to this folder which are owned by root. + # Using a relative path, this the container will not write to the folder on the host. + # So do not use an absolute path, otherwise we cannot remove this folder on the host. + AUTH_CONFIG="C$(unique_id)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_REGISTRY_CONFIGS_FILE() { local TEST_NAME="${1}" @@ -204,14 +210,14 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}" - The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}" + The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}" The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}" The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}" The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" @@ -220,7 +226,7 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" - The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME}" # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service. The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*" @@ -245,7 +251,7 @@ Describe 'login' TEST_NAME="test_login_external_config" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - CONFIG="$(mktemp -d)" + AUTH_CONFIG="$(mktemp -d)" TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1 test_login_external_config() { local TEST_NAME="${1}" @@ -276,7 +282,7 @@ Describe 'login' BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_login_external_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" + When run test_login_external_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index d79ca32..3a660d6 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -213,7 +213,7 @@ Describe 'manifest-command' The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false" "coverage:true" + Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:true" "coverage:true" TEST_NAME="test_MANIFEST_CMD_unsupported_cmd" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") @@ -260,7 +260,7 @@ Describe 'manifest-command' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_MANIFEST_CMD_failure" "container_test:false" "coverage:true" + Describe "test_MANIFEST_CMD_failure" "container_test:true" "coverage:true" TEST_NAME="test_MANIFEST_CMD_failure" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh index 02dacdd..2871b21 100644 --- a/tests/gantry_notify_spec.sh +++ b/tests/gantry_notify_spec.sh @@ -320,7 +320,7 @@ Describe 'notify' The stderr should satisfy spec_expect_message "${SKIP_SENDING_NOTIFICATION}" End End - Describe "test_notify_on_change_errors" "container_test:false" "coverage:true" + Describe "test_notify_on_change_errors" "container_test:true" "coverage:true" TEST_NAME="test_notify_on_change_errors" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh index b1f77ab..53b3e32 100644 --- a/tests/gantry_parallel_spec.sh +++ b/tests/gantry_parallel_spec.sh @@ -132,7 +132,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true" + Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:true" "coverage:true" TEST_NAME="test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") @@ -176,7 +176,7 @@ Describe 'service-parallel' The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true" + Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:true" "coverage:true" TEST_NAME="test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh index 8d97030..4906839 100644 --- a/tests/gantry_update_options_spec.sh +++ b/tests/gantry_update_options_spec.sh @@ -140,7 +140,7 @@ Describe 'update-options' The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}" End End - Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true" + Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:true" "coverage:true" TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 0d4be29..3b86d41 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -15,8 +15,6 @@ # along with this program. If not, see . # -set -a - # Constant strings for checks. # START_WITHOUT_A_SQUARE_BRACKET ignores color codes. Use test_log not to trigger this check. export START_WITHOUT_A_SQUARE_BRACKET="^(?!(?:\x1b\[[0-9;]*[mG])?\[)" @@ -76,6 +74,8 @@ export TEST_SERVICE_IMAGE="alpine:latest" test_log() { echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^NONE$" && return 0; + echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^ERROR$" && return 0; + echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^WARN$" && return 0; [ -n "${GANTRY_IMAGES_TO_REMOVE}" ] && echo "${*}" >&2 && return 0; echo "[$(date -Iseconds)] Test: ${*}" >&2 } @@ -422,6 +422,7 @@ reset_gantry_env() { local SERVICE_NAME="${1}" export DOCKER_CONFIG= export DOCKER_HOST= + export GANTRY_TEST_DOCKER_HOST= export GANTRY_LOG_LEVEL="DEBUG" export GANTRY_NODE_NAME= export GANTRY_POST_RUN_CMD= @@ -688,18 +689,10 @@ _add_htpasswd() { } _wait_service_state() { - local SERVICE_NAME="${1}" - local STATE="${2}" - local TRIES=0 - local MAX_RETRIES=120 - while ! docker service ps --format "{{.CurrentState}}" "${SERVICE_NAME}" | grep -q "${STATE}"; do - if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then - echo "_wait_service_state Reach MAX_RETRIES ${MAX_RETRIES}" >&2 - return 1 - fi - TRIES=$((TRIES+1)) - sleep 1 - done + local SERVICE_NAME="${1}"; + local WANT_STATE="${2}"; + local TIMEOUT_SECONDS="${3}"; + wait_service_state "${SERVICE_NAME}" "${WANT_STATE}" "${TIMEOUT_SECONDS}" 1>/dev/null 2>&1 } _correct_test_service_name() { @@ -746,8 +739,8 @@ start_replicated_service() { $(_location_constraints) \ --mode=replicated \ --detach=true \ - "${IMAGE_WITH_TAG}" - _wait_service_state "${SERVICE_NAME}" "Running" + "${IMAGE_WITH_TAG}"; + _wait_service_state "${SERVICE_NAME}" "Running" 120 } start_multiple_replicated_services() { @@ -808,9 +801,9 @@ _start_replicated_job() { $(_location_constraints) \ --mode=replicated-job \ --detach=true \ - "${IMAGE_WITH_TAG}" + "${IMAGE_WITH_TAG}"; # wait until the job is running - _wait_service_state "${SERVICE_NAME}" "Running" + _wait_service_state "${SERVICE_NAME}" "Running" 120 } stop_service() { @@ -879,15 +872,37 @@ _add_file_to_mount_options() { echo "${MOUNT_OPTIONS}" } -_run_gantry_container() { +stop_gantry() { local STACK="${1}" + local PID="${2}" + local RETURN_VALUE=0 local SUT_REPO_TAG= SUT_REPO_TAG="$(_get_sut_image)" - if [ -z "${SUT_REPO_TAG}" ]; then - return 1 + if [ -n "${SUT_REPO_TAG}" ]; then + local SERVICE_NAME= + SERVICE_NAME="gantry-test-SUT-${STACK}" + SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}") + local CMD_OUTPUT= + docker service logs --raw "${SERVICE_NAME}" + if ! CMD_OUTPUT=$(docker service rm "${SERVICE_NAME}" 2>&1); then + echo "Failed to remove service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2 + RETURN_VALUE=1 + fi fi + if [ -n "${PID}" ]; then + kill "${PID}" + RETURN_VALUE=$? + fi + return "${RETURN_VALUE}" +} + +_run_gantry_container() { + local STACK="${1}" + local SUT_REPO_TAG="${2}" + pull_image_if_not_exist "${SUT_REPO_TAG}" local SERVICE_NAME= - SERVICE_NAME="gantry-test-SUT-$(unique_id)" + SERVICE_NAME="gantry-test-SUT-${STACK}" + SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}") docker service rm "${SERVICE_NAME}" >/dev/null 2>&1; local MOUNT_OPTIONS= MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${DOCKER_CONFIG}") @@ -902,12 +917,13 @@ _run_gantry_container() { # SC2086 (info): Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 if ! CMD_OUTPUT=$(docker service create --name "${SERVICE_NAME}" \ + --detach=true \ --mode replicated-job --restart-condition=none --network host \ --constraint "node.role==manager" \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ ${MOUNT_OPTIONS} \ --env "DOCKER_CONFIG=${DOCKER_CONFIG}" \ - --env "DOCKER_HOST=${DOCKER_HOST}" \ + --env "DOCKER_HOST=${GANTRY_TEST_DOCKER_HOST}" \ --env "GANTRY_LOG_LEVEL=${GANTRY_LOG_LEVEL}" \ --env "GANTRY_NODE_NAME=${GANTRY_NODE_NAME}" \ --env "GANTRY_POST_RUN_CMD=${GANTRY_POST_RUN_CMD}" \ @@ -948,22 +964,27 @@ _run_gantry_container() { echo "Failed to create service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2 RETURN_VALUE=1 fi - docker service logs --raw "${SERVICE_NAME}" - if ! CMD_OUTPUT=$(docker service rm "${SERVICE_NAME}" 2>&1); then - echo "Failed to remove service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2 - RETURN_VALUE=1 - fi + _wait_service_state "${SERVICE_NAME}" "Complete" 120 + RETURN_VALUE=$? + stop_gantry "${STACK}" || return 1 return "${RETURN_VALUE}" } run_gantry() { local STACK="${1}" - if _run_gantry_container "${STACK}"; then - return 0 + local SUT_REPO_TAG= + SUT_REPO_TAG="$(_get_sut_image)" + if [ -n "${SUT_REPO_TAG}" ]; then + _run_gantry_container "${STACK}" "${SUT_REPO_TAG}" + return $? fi + local OLD_DOCKER_HOST="${DOCKER_HOST}" + [ -n "${GANTRY_TEST_DOCKER_HOST}" ] && export DOCKER_HOST="${GANTRY_TEST_DOCKER_HOST}" + local RETURN_VALUE= local ENTRYPOINT= ENTRYPOINT=$(_get_entrypoint) || return 1 ${ENTRYPOINT} "${STACK}" + RETURN_VALUE=$? + export DOCKER_HOST="${OLD_DOCKER_HOST}" + return "${RETURN_VALUE}" } - -set +a