Skip to content

Commit

Permalink
[tests] allow more tests to run with a container.
Browse files Browse the repository at this point in the history
  • Loading branch information
@shizunge
shizunge committed Nov 25, 2024
1 parent d6bf10a commit 90f0865
Show file tree
Hide file tree
Showing 12 changed files with 194 additions and 127 deletions.
27 changes: 19 additions & 8 deletions src/lib-common.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,12 @@ _random_string() {

_pipe_name() {
local BASE_NAME="${1:-pipe-base-name}"
local RANDOM_STR=
RANDOM_STR=$(_random_string)
local PID=$$
local TIMESTAMP=
TIMESTAMP=$(date +%s)
local PIPE_NAME="/tmp/${BASE_NAME}-$$-${TIMESTAMP}-${RANDOM_STR}"
local RANDOM_STR=
RANDOM_STR=$(_random_string)
local PIPE_NAME="/tmp/${BASE_NAME}-${PID}-${TIMESTAMP}-${RANDOM_STR}"
echo "${PIPE_NAME}"
}

Expand Down Expand Up @@ -557,25 +558,35 @@ _all_tasks_reach_state() {
return 0
}

# Usage: wait_service_state <SERVICE_NAME> <WANT_STATE>
# Usage: wait_service_state <SERVICE_NAME> <WANT_STATE> [timeout in seconds]
# Wait for the service, usually a global job or a replicated job,
# to reach either running or complete state.
# Valid WANT_STATE includes "Running" and "Complete"
# When the WANT_STATE is complete, the function returns immediately
# when any of the tasks of the service fails.
# In case of task failing, the function returns a non-zero value.
# When the WANT_STATE is complete, the function returns immediately when any of the tasks of the service fails.
# In case of task failing, the function returns the first failing task's return value.
wait_service_state() {
local SERVICE_NAME="${1}";
local WANT_STATE="${2}";
local TIMEOUT_SECONDS="${3}";
local CHECK_FAILURES=false
[ "${WANT_STATE}" = "Complete" ] && CHECK_FAILURES=true
local SLEEP_SECONDS=1
local DOCKER_CMD_ERROR=1
local START_TIME=
START_TIME=$(date +%s)
local RETURN_VALUE=0
local DOCKER_CMD_ERROR=1
local STATES=
while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do
DOCKER_CMD_ERROR=0
RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break
local SECONDS_ELAPSED=
if is_number "${TIMEOUT_SECONDS}" \
&& SECONDS_ELAPSED=$(first_minus_second "$(date +%s)" "${START_TIME}") \
&& [ "${SECONDS_ELAPSED}" -ge "${TIMEOUT_SECONDS}" ]; then
log ERROR "wait_service_state ${SERVICE_NAME} ${WANT_STATE} timeout after ${SECONDS_ELAPSED}s."
RETURN_VALUE=2
break
fi
sleep "${SLEEP_SECONDS}"
DOCKER_CMD_ERROR=1
done
Expand Down
7 changes: 3 additions & 4 deletions tests/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,13 @@ bash shellspec --jobs 50

To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) installed):
```
bash shellspec --kcov --tag coverage:true
bash shellspec --kcov --tag "coverage:true"
```

If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
```
export GANTRY_TEST_CONTAINER_REPO_TAG=<gantry image>:<tag>
bash shellspec --tag "container_test:true" "coverage:true"
bash shellspec --tag "container_test:true"
```

> NOTE: Negative tests will hang when testing a *Gantry* container, which may be due to a bug in shellspec. So when testing *Gantry* images, we should run only tests with tag `container_test:true`.
> NOTE: Some tests will fail when testing a *Gantry* container. So when testing *Gantry* images, we should run only tests with tag `container_test:true`.
21 changes: 10 additions & 11 deletions tests/gantry_common_options_spec.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,19 @@ Describe 'common-options'
SUITE_NAME="common-options"
BeforeAll "initialize_all_tests ${SUITE_NAME}"
AfterAll "finish_all_tests ${SUITE_NAME}"
Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true"
Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:true" "coverage:true"
TEST_NAME="test_common_DOCKER_HOST_not_swarm_manager"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
test_common_DOCKER_HOST_not_swarm_manager() {
local TEST_NAME="${1}"
local SERVICE_NAME="${2}"
reset_gantry_env "${SERVICE_NAME}"
export DOCKER_HOST="8.8.8.8:53"
export GANTRY_TEST_DOCKER_HOST="8.8.8.8:53"
local RETURN_VALUE=0
run_gantry "${TEST_NAME}"
RETURN_VALUE="${?}"
export DOCKER_HOST=
export GANTRY_TEST_DOCKER_HOST=
return "${RETURN_VALUE}"
}
BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
Expand Down Expand Up @@ -93,10 +93,12 @@ Describe 'common-options'
End
End
# Do not run test_common_no_new_env with the kcov, which alters the environment variables.
Describe "test_common_no_new_env" "container_test:false" "coverage:false"
Describe "test_common_no_new_env" "container_test:true" "coverage:false"
# Check there is no new variable set,
# to avoid errors like https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
# We don't need to run this test using containers because we check env on the host, while the container test set env inside the container.
#
# It makes no sense to run run this test using containers because we check env on the host, while the container test set env inside the container.
# But it should not failed with a container. We are just testing GANTRY_LOG_LEVEL=WARN.
TEST_NAME="test_common_no_new_env"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
Expand Down Expand Up @@ -195,10 +197,7 @@ Describe 'common-options'
The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
End
End
# run_gantry prints logs after gantry exists, while testing a container.
# In thes test, gantry never exit, but will be killed, thus there is no log.
# Therefore we disable the container test for this test.
Describe "test_common_SLEEP_SECONDS" "container_test:false" "coverage:true"
Describe "test_common_SLEEP_SECONDS" "container_test:true" "coverage:true"
TEST_NAME="test_common_SLEEP_SECONDS"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
Expand All @@ -210,7 +209,7 @@ Describe 'common-options'
run_gantry "${TEST_NAME}" &
local PID="${!}"
sleep $((GANTRY_SLEEP_SECONDS*3+1))
kill "${PID}"
stop_gantry "${TEST_NAME}" "${PID}"
}
BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
Expand Down Expand Up @@ -249,7 +248,7 @@ Describe 'common-options'
The stderr should satisfy spec_expect_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
End
End
Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false" "coverage:true"
Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:true" "coverage:true"
TEST_NAME="test_common_SLEEP_SECONDS_not_a_number"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
Expand Down
4 changes: 2 additions & 2 deletions tests/gantry_filters_spec.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ Describe 'filters'
SUITE_NAME="filters"
BeforeAll "initialize_all_tests ${SUITE_NAME}"
AfterAll "finish_all_tests ${SUITE_NAME}"
Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
Describe "test_SERVICES_FILTERS_bad" "container_test:true" "coverage:true"
TEST_NAME="test_SERVICES_FILTERS_bad"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
Expand Down Expand Up @@ -173,7 +173,7 @@ Describe 'filters'
The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
End
End
Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:true" "coverage:true"
TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
Expand Down
35 changes: 22 additions & 13 deletions tests/gantry_login_docker_config_spec.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ Describe 'login_docker_config'
TEST_NAME="test_login_docker_config_no_label"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
CONFIG="C$(unique_id)"
# When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
# Using a relative path, this the container will not write to the folder on the host.
# So do not use an absolute path, otherwise we cannot remove this folder on the host.
AUTH_CONFIG="C$(unique_id)"
TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
test_login_docker_config_no_label() {
local TEST_NAME="${1}"
Expand Down Expand Up @@ -68,14 +71,14 @@ Describe 'login_docker_config'
BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
It 'run_test'
When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
The status should be success
The stdout should satisfy display_output
The stdout should satisfy spec_expect_no_message ".+"
The stderr should satisfy display_output
The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
Expand Down Expand Up @@ -110,7 +113,10 @@ Describe 'login_docker_config'
TEST_NAME="test_login_docker_config_default_config"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
CONFIG="C$(unique_id)"
# When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
# Using a relative path, this the container will not write to the folder on the host.
# So do not use an absolute path, otherwise we cannot remove this folder on the host.
AUTH_CONFIG="C$(unique_id)"
TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
test_login_docker_config_default_config() {
local TEST_NAME="${1}"
Expand Down Expand Up @@ -141,7 +147,7 @@ Describe 'login_docker_config'
BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
It 'run_test'
When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
The status should be success
The stdout should satisfy display_output
The stdout should satisfy spec_expect_no_message ".+"
Expand Down Expand Up @@ -177,14 +183,17 @@ Describe 'login_docker_config'
The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}"
End
End
Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
Describe "test_login_docker_config_label_override" "container_test:true" "coverage:true"
TEST_NAME="test_login_docker_config_label_override"
IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
SERVICE_NAME0="${SERVICE_NAME}-0"
SERVICE_NAME1="${SERVICE_NAME}-1"
SERVICE_NAME2="${SERVICE_NAME}-2"
CONFIG="C$(unique_id)"
# When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
# Using a relative path, this the container will not write to the folder on the host.
# So do not use an absolute path, otherwise we cannot remove this folder on the host.
AUTH_CONFIG="C$(unique_id)"
TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
test_start() {
local TEST_NAME="${1}"
Expand Down Expand Up @@ -241,21 +250,21 @@ Describe 'login_docker_config'
BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
It 'run_test'
When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
The status should be failure
The stdout should satisfy display_output
The stdout should satisfy spec_expect_no_message ".+"
The stderr should satisfy display_output
The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
The stderr should satisfy spec_expect_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
The stderr should satisfy spec_expect_message "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
The stderr should satisfy spec_expect_message "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
# Check warnings
The stderr should satisfy spec_expect_message "${THERE_ARE_NUM_CONFIGURATIONS}.*"
The stderr should satisfy spec_expect_message "${USER_LOGGED_INTO_DEFAULT}.*"
The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME0}"
The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}"
The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME2}"
The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME0}"
The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}"
The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME2}"
The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*"
The stderr should satisfy spec_expect_message "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}"
The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
Expand Down
Loading

0 comments on commit 90f0865

Please sign in to comment.