diff --git a/README.md b/README.md index 5e918a4..4a9bb23 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ You can configure the most behaviors of *Gantry* via environment variables. | Environment Variable | Default | Description | |-----------------------|---------|-------------| | GANTRY_SERVICES_EXCLUDED | | A space separated list of services names that are excluded from updating. | -| GANTRY_SERVICES_EXCLUDED_FILTERS | `label=gantry.services.excluded=true` | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter), e.g. `label=project=project-a`. Exclude services which match the given filters from updating. The default value allows you to add label `gantry.services.excluded=true` to services to exclude them from updating. Note that multiple filters will be logical **ANDED**. | -| GANTRY_SERVICES_FILTERS | | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter) that are accepted by `docker service ls --filter` to select services to update, e.g. `label=project=project-a`. Note that multiple filters will be logical **ANDED**. Also see [How to filters multiple services by name](docs/faq.md#how-to-filters-multiple-services-by-name). | +| GANTRY_SERVICES_EXCLUDED_FILTERS | `label=gantry.services.excluded=true` | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter), e.g. `label=project=project-a`. Exclude services which match the given filters from updating. The default value allows you to add label `gantry.services.excluded=true` to services to exclude them from updating. Note that multiple filters will be logical **ANDED**. An empty string means no filters, as a result *Gantry* will not exclude any services. | +| GANTRY_SERVICES_FILTERS | | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter) that are accepted by `docker service ls --filter` to select services to update, e.g. `label=project=project-a`. Note that multiple filters will be logical **ANDED**. An empty string means no filters, as a result *Gantry* will update all services. Also see [How to filters multiple services by name](docs/faq.md#how-to-filters-multiple-services-by-name). | > NOTE: *Gantry* reads labels on the services not on the containers. The labels need to go to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section, if you are using docker compose files to setup your services. diff --git a/docs/authentication.md b/docs/authentication.md index 08e0073..bcbf6aa 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -30,17 +30,19 @@ If the images of services are hosted on multiple registries that are required au You can use `GANTRY_REGISTRY_CONFIGS_FILE` together with other authentication environment variables. -You can login to multiple registries using the same Docker configuration. However if you login to the same registry with different user names for different services, you should use different Docker configurations. +You can login to multiple registries using the same Docker configuration. For example you can set all the configurations to the default Docker configuration location `${HOME}/.docker/`. However if you login to the same registry with different user names for different services, you need to use different Docker configurations. -### Select Docker configurations for services +### Selecting Docker configurations for services -If you login to a single registry using `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_HOST` without setting `GANTRY_REGISTRY_CONFIG`, the default Docker configuration is used. You don't need to set anything extra for authentication. +If you login to a single registry using `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_HOST` **without** setting `GANTRY_REGISTRY_CONFIG`, the default Docker configuration is used. When using the default Docker configuration, you don't need to set anything extra for authentication. - *Gantry* creates or updates the docker configurations based on the locations set via `GANTRY_REGISTRY_CONFIG`, `GANTRY_REGISTRY_CONFIG_FILE` or `GANTRY_REGISTRY_CONFIGS_FILE`. + *Gantry* creates or updates the docker configurations based on the locations set via `GANTRY_REGISTRY_CONFIG`, `GANTRY_REGISTRY_CONFIG_FILE` or `GANTRY_REGISTRY_CONFIGS_FILE`. They could be same as the the default Docker configuration location. When using the default Docker configuration, you don't need to set anything extra for authentication. - You can use environment variable [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables) to apply the same Docker configuration to all docker commands, i.e. to all services. +The default Docker configuration location is `/root/.docker/` inside the container created based on the image built from this repository, because the default user is `root`. You can use environment variable [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables) to explicitly set the default docker configuration location, which applies to all docker commands, i.e. to all services. -You need to add label `gantry.auth.config=` to particular services to tell which Docker configuration to use for authentication, when you use different configurations for different registries. When *Gantry* finds the label `gantry.auth.config=` on services, it adds `--config ` to the Docker commands for the corresponding services. +Optionally you can use different configurations for different services, for example when you want to login to the same registry with different user names. In this case, besides using different configuration values in `GANTRY_REGISTRY_CONFIG` and `GANTRY_REGISTRY_CONFIG_FILE`, you need to add the label `gantry.auth.config=` on the particular services to tell which Docker configuration to use for authentication. When *Gantry* finds the label `gantry.auth.config=` on services, it adds `--config ` to the Docker commands for the corresponding services to overrides the default configuration location. + +### Adding `--with-registry-auth` *Gantry* automatically adds `--with-registry-auth` to the `docker service update` command for services for the following cases. @@ -49,9 +51,9 @@ You need to add label `gantry.auth.config=` to particular service * when `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` are set, while `GANTRY_REGISTRY_CONFIG` is empty. * when the configuration from `GANTRY_REGISTRY_CONFIG` or `GANTRY_REGISTRY_CONFIGS_FILE` is same as the default Docker configuration location `${HOME}/.docker/` or the location specified by `DOCKER_CONFIG`. -You can manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` if it is not added automatically for your case. Without `--with-registry-auth`, the service will be [updated to an image without digest](https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336), and you might get a warning "*image \ could not be accessed on a registry to record its digest. Each node will access \ independently, possibly leading to different nodes running different versions of the image.*" +You can manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` if it is not added automatically for your case. When `--with-registry-auth` is missing but the registry requires authentication, the service will be [updated to an image without digest](https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336), and you will get a warning *"image \ could not be accessed on a registry to record its digest. Each node will access \ independently, possibly leading to different nodes running different versions of the image."* -### Use an existing Docker configuration +### Using an existing Docker configuration You can use an existing Docker configuration from the host machines for authorization when you run *Gantry* as a Docker service. You need to do the followings. @@ -60,4 +62,4 @@ You can use an existing Docker configuration from the host machines for authoriz * Set the environment variable `DOCKER_CONFIG` on the *Gantry* container to specify the location of the Docker configuration folder inside the container. You can skip this step when you mount the folder to the default Docker configuration location `/root/.docker/` inside the container. * Add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` manually. -> Note that [`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/) writes data to the Docker configuration folder `${DOCKER_CONFIG}/buildx`, which therefore needs to be writable. You can set `GANTRY_MANIFEST_CMD` to `manifest` to avoid writing to the Docker configuration folder. +> Note that [`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/) writes data to the Docker configuration folder `${DOCKER_CONFIG}/buildx`, which therefore needs to be writable. If you want to use `buildx` and mount the configuration files read-only, you could just mount the file `config.json` and leave the folder writeable. If you have to mount the entire folder read-only, you can set `GANTRY_MANIFEST_CMD` to `manifest` to avoid writing to the Docker configuration folder. Also see [Which `GANTRY_MANIFEST_CMD` to use](../docs/faq.md#which-gantry_manifest_cmd-to-use). diff --git a/docs/faq.md b/docs/faq.md index 303daad..2e733bc 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -18,6 +18,8 @@ It will not work by setting multiple filters with different names, because filte To filter multiple services, you can set a label on each service then let *Gantry* filter on that label via `GANTRY_SERVICES_FILTERS`. Or you can run multiple *Gantry* instances. +Setting `GANTRY_SERVICES_FILTERS` to an empty string means no filters, as a result *Gantry* will update all services. + ### How to run *Gantry* on a cron schedule? You can start *Gantry* as a docker swarm service and use [`swarm-cronjob`](https://github.com/crazy-max/swarm-cronjob) to run it at a given time. When use `swarm-cronjob`, you need to set `GANTRY_SLEEP_SECONDS` to 0. See the [example](../examples/cronjob). @@ -36,7 +38,7 @@ Before updating a service, *Gantry* will try to obtain the image's meta data to `manifest` is kept for debugging purpose. The only known advantage of [`docker manifest inspect`](https://docs.docker.com/engine/reference/commandline/manifest_inspect/) is that it does not require the write permission to the [Docker configuration folder](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files). If the Docker configuration folder is read-only, you need to use `manifest` to avoid permission deny errors. -`none` can be used to disable the image inspection. One use case of `none` is that you want to add `--force` to the `docker service update` command via `GANTRY_UPDATE_OPTIONS`, which updates the services even if there is nothing changed. Another use case is to debug image inspection. Please report the bug through a [GitHub issue](https://github.com/shizunge/gantry/issues), thanks. +`none` can be used to disable the image inspection. One use case of `none` is that you want to add `--no-resolve-image` and `--force` to the `docker service update` command via `GANTRY_UPDATE_OPTIONS`, which allows you to update services using local images. `none` can also be used to debug image inspection. ### Can *Gantry* report Docker Hub rate for non-anonymous account? diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh index f573b8f..a3a926d 100755 --- a/src/lib-gantry.sh +++ b/src/lib-gantry.sh @@ -1037,13 +1037,17 @@ _update_single_service() { _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_FAILED "${SERVICE_NAME}" return 1 fi - local TIME_ELAPSED= - TIME_ELAPSED=$(time_elapsed_since "${START_TIME}") local PREVIOUS_IMAGE= - local CURRENT_IMAGE= PREVIOUS_IMAGE=$(_get_service_previous_image "${SERVICE_NAME}") + PREVIOUS_DIGEST=$(extract_string "${PREVIOUS_IMAGE}" '@' 2) + [ -z "${PREVIOUS_DIGEST}" ] && log DEBUG "After updating, the previous image ${PREVIOUS_IMAGE} of ${SERVICE_NAME} does not have a digest." + local CURRENT_IMAGE= CURRENT_IMAGE=$(_get_service_image "${SERVICE_NAME}") - if [ "${PREVIOUS_IMAGE}" = "${CURRENT_IMAGE}" ]; then + CURRENT_DIGEST=$(extract_string "${CURRENT_IMAGE}" '@' 2) + [ -z "${CURRENT_DIGEST}" ] && log WARN "After updating, the current image ${CURRENT_IMAGE} of ${SERVICE_NAME} does not have a digest." + local TIME_ELAPSED= + TIME_ELAPSED=$(time_elapsed_since "${START_TIME}") + if [ -n "${CURRENT_DIGEST}" ] && [ "${PREVIOUS_DIGEST}" = "${CURRENT_DIGEST}" ]; then log INFO "No updates for ${SERVICE_NAME}. Use ${TIME_ELAPSED}." return 0 fi diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh index f46fc32..003bc06 100644 --- a/tests/gantry_login_spec.sh +++ b/tests/gantry_login_spec.sh @@ -82,6 +82,7 @@ Describe 'login' The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*automatically.*${SERVICE_NAME}" The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*specified by user.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -152,6 +153,7 @@ Describe 'login' # Gantry adds --with-registry-auth for using the default configuration. The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -230,6 +232,7 @@ Describe 'login' # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service. The stderr should satisfy spec_expect_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -302,6 +305,8 @@ Describe 'login' The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*" # Check the warning due to missing --with-registry-auth. The stderr should satisfy spec_expect_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${AFTER_UPDATING_PREVIOUS_IMAGE}" + The stderr should satisfy spec_expect_message "${AFTER_UPDATING_CURRENT_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh index 8fee3c2..5e2e247 100644 --- a/tests/gantry_manifest_spec.sh +++ b/tests/gantry_manifest_spec.sh @@ -19,11 +19,11 @@ Describe 'manifest-command' SUITE_NAME="manifest-command" BeforeAll "initialize_all_tests ${SUITE_NAME}" AfterAll "finish_all_tests ${SUITE_NAME}" - Describe "test_MANIFEST_CMD_none" - TEST_NAME="test_MANIFEST_CMD_none" + Describe "test_MANIFEST_CMD_none_force" + TEST_NAME="test_MANIFEST_CMD_none_force" IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") - test_MANIFEST_CMD_none() { + test_MANIFEST_CMD_none_force() { local TEST_NAME="${1}" local SERVICE_NAME="${2}" reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}" @@ -34,13 +34,13 @@ Describe 'manifest-command' BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" It 'run_test' - When run test_MANIFEST_CMD_none "${TEST_NAME}" "${SERVICE_NAME}" + When run test_MANIFEST_CMD_none_force "${TEST_NAME}" "${SERVICE_NAME}" The status should be success The stdout should satisfy display_output The stdout should satisfy spec_expect_no_message ".+" The stderr should satisfy display_output The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}" - # Do not set GANTRY_SERVICES_SELF, it should be set autoamtically + # Do not set GANTRY_SERVICES_SELF, it should be set automatically. # If we are not testing gantry inside a container, it should failed to find the service name. # To test gantry container, we need to use run_gantry_container. The stderr should satisfy spec_expect_no_message ".*GANTRY_SERVICES_SELF.*" diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh index 9c0fd11..a0ff061 100644 --- a/tests/gantry_service_single_spec.sh +++ b/tests/gantry_service_single_spec.sh @@ -47,6 +47,7 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}" The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}" The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}" + The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -93,6 +94,7 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}" The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}" The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}" + The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -119,6 +121,7 @@ Describe 'service-single-service' local TEST_NAME="${1}" local IMAGE_WITH_TAG="${2}" local SERVICE_NAME="${3}" + initialize_test "${TEST_NAME}" # Start a service with image not available on the registry, the digest will not be available. build_test_image "${IMAGE_WITH_TAG}" start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 2>&1 @@ -149,6 +152,8 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}" The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}" The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}" + The stderr should satisfy spec_expect_message "${AFTER_UPDATING_PREVIOUS_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}" + The stderr should satisfy spec_expect_no_message "${AFTER_UPDATING_CURRENT_IMAGE}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" @@ -161,7 +166,72 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" - # Failed to removing the old image due to it has no digest. + # Failed to remove the old image due to it has no digest. + The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" + The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}" + End + End + Describe "test_new_image_local" + TEST_NAME="test_new_image_local" + IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}") + SERVICE_NAME=$(get_test_service_name "${TEST_NAME}") + test_start() { + local TEST_NAME="${1}" + local IMAGE_WITH_TAG="${2}" + local SERVICE_NAME="${3}" + initialize_test "${TEST_NAME}" + # Start a service with image not available on the registry, the digest will not be available. + build_test_image "${IMAGE_WITH_TAG}" + start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 2>&1 + # Build a new local image. Do not push to registry to test `--no-resolve-image`. + build_test_image "${IMAGE_WITH_TAG}" + } + test_new_image_local() { + local TEST_NAME="${1}" + local SERVICE_NAME="${2}" + reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}" + # The image is not pushed to the registry. + export GANTRY_MANIFEST_CMD="none" + # Because both new and old images have no digest, `docker service update` thinks there is nothing to update, thus it will not run the update without `--force`. + export GANTRY_UPDATE_OPTIONS="--force" + run_gantry "${SUITE_NAME}" "${TEST_NAME}" + } + BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}" + It 'run_test' + When run test_new_image_local "${TEST_NAME}" "${SERVICE_NAME}" + The status should be success + The stdout should satisfy display_output + The stdout should satisfy spec_expect_no_message ".+" + The stderr should satisfy display_output + # Gantry is still trying to update the service. + # It will see a new local image. + The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_MANIFEST_CMD_IS_NONE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATING}" + The stderr should satisfy spec_expect_message "${ADDING_OPTIONS}.*--force.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}" + The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}" + The stderr should satisfy spec_expect_message "${AFTER_UPDATING_PREVIOUS_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}" + The stderr should satisfy spec_expect_message "${AFTER_UPDATING_CURRENT_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}" + The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}" + The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_message "${NUM_SERVICES_UPDATED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}" + The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}" + The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}" + The stderr should satisfy spec_expect_message "${REMOVING_NUM_IMAGES}" + The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}" + # Failed to remove the image because it should be used by the service. The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}" The stderr should satisfy spec_expect_message "${DONE_REMOVING_IMAGES}" @@ -200,6 +270,7 @@ Describe 'service-single-service' The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}" The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}" The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}" + The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}" The stderr should satisfy spec_expect_message "${UPDATED}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}" The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}" diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh index 131a8a3..cd2611e 100644 --- a/tests/spec_gantry_test_helper.sh +++ b/tests/spec_gantry_test_helper.sh @@ -50,6 +50,9 @@ export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)" export NUM_SERVICES_NO_NEW_IMAGES="No new images for [0-9]+ service\(s\)" export NUM_SERVICES_UPDATING="Updating [0-9]+ service\(s\)" export NO_UPDATES="No updates" +export AFTER_UPDATING_CURRENT_IMAGE="After updating, the current image" +export AFTER_UPDATING_PREVIOUS_IMAGE="After updating, the previous image" +export DOES_NOT_HAVE_A_DIGEST="does not have a digest" export UPDATED="Updated" export ROLLING_BACK="Rolling back" export FAILED_TO_ROLLBACK="Failed to roll back" @@ -642,14 +645,27 @@ build_test_image() { if [ -n "${EXIT_SECONDS}" ] && [ "${EXIT_SECONDS}" -gt "0" ]; then EXIT_CMD="sleep ${EXIT_SECONDS};" fi - local FILE= - FILE=$(make_test_temp_file) - echo "FROM $(_get_test_service_image)" > "${FILE}" - echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}" - pull_image_if_not_exist "$(_get_test_service_image)" - echo "Building image ${IMAGE_WITH_TAG} from ${FILE}" - docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . - rm "${FILE}" + local RETURN_VALUE=1 + local TRIES=0 + local MAX_RETRIES=60 + while [ "${RETURN_VALUE}" != "0" ]; do + if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then + echo "build_test_image Reach MAX_RETRIES ${MAX_RETRIES}" >&2 + return 1 + fi + TRIES=$((TRIES+1)) + local FILE= + FILE=$(make_test_temp_file) + echo "FROM $(_get_test_service_image)" > "${FILE}" + echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}" + pull_image_if_not_exist "$(_get_test_service_image)" + echo "Building image ${IMAGE_WITH_TAG} from ${FILE}" + docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . 2>&1 + RETURN_VALUE=$? + rm "${FILE}" + [ "${RETURN_VALUE}" != "0" ] && sleep 1 + done + return 0 } build_and_push_test_image() { @@ -666,7 +682,7 @@ build_and_push_test_image() { prune_local_test_image() { local IMAGE_WITH_TAG="${1}" echo "Removing image ${IMAGE_WITH_TAG}" - docker image rm "${IMAGE_WITH_TAG}" --force + docker image rm "${IMAGE_WITH_TAG}" --force 2>&1 } docker_service_update() {