From 4f1835ca772b36bd15f30b7787cf14e5d0cfc779 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Sat, 16 Nov 2024 23:48:58 -0800
Subject: [PATCH 01/27] [lib-common] add extract_string

---
 src/lib-common.sh | 46 +++++++++++++++++++++++++++++++++++-----------
 src/lib-gantry.sh | 18 ++++++++----------
 2 files changed, 43 insertions(+), 21 deletions(-)

diff --git a/src/lib-common.sh b/src/lib-common.sh
index c1ddfed..cd36447 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -34,6 +34,30 @@ grep_q_i() {
   return "${GREP_RETURN}"
 }
 
+# Extract ${POSITION}th part of the string from a single line ${SINGLE_LINE}, separated by ${DELIMITER}.
+extract_string() {
+  local SINGLE_LINE="${1}"
+  local DELIMITER="${2}"
+  local POSITION="${3}"
+  # When the input contains no ${DELIMITER}, there are the expect outputs
+  # * ${POSITION} is 1 -> the ${SINGLE_LINE}
+  # * Other ${POSITION} -> an empty string
+  # The following command(s) won't work if we do not add the ${DELIMITER} to the end of ${SINGLE_LINE}
+  # * `echo "${SINGLE_LINE}" | cut -s -d "${DELIMITER}" -f 1`: actually return an empty string.
+  # * `echo "${SINGLE_LINE}" | cut -d "${DELIMITER}" -f 2`: actually returns the any line that contains no delimiter.
+  # We add a ${DELIMITER} to the echo command to ensure the string contains at least one ${DELIMITER},
+  # to help us get the expected output above.
+  # When the input contains a ${DELIMITER}, for the following command(s)
+  # * `echo "${SINGLE_LINE}${DELIMITER}" | cut -d "${DELIMITER}" -f 2-`
+  # we do not want to see a ${DELIMITER} at the end of the ouput,
+  # therefore we do not always add the ${DELIMITER} to the end of ${SINGLE_LINE}.
+  local ECHO_STRING="${SINGLE_LINE}"
+  if ! echo "${SINGLE_LINE}" | grep_q "${DELIMITER}"; then
+    ECHO_STRING="${SINGLE_LINE}${DELIMITER}"
+  fi
+  echo "${ECHO_STRING}" | cut -d "${DELIMITER}" -f "${POSITION}"
+}
+
 # echo the number of the log level.
 # return 0 if LEVEL is supported.
 # return 1 if LEVLE is unsupported.
@@ -150,19 +174,19 @@ _log_docker_node() {
 _log_docker_line() {
   local LEVEL="INFO";
   local TIME_DOCKER TIME TASK_NODE SCOPE NODE MESSAGE SPACE FIRST_WORD
-  TIME_DOCKER=$(echo "${*} " | cut -d ' ' -f 1);
+  TIME_DOCKER=$(extract_string "${*}" ' ' 1)
   TIME=$(_log_docker_time "${TIME_DOCKER}")
-  TASK_NODE=$(echo "${*} " | cut -d ' ' -f 2)
+  TASK_NODE=$(extract_string "${*}" ' ' 2)
   SCOPE=$(_log_docker_scope "${TASK_NODE}");
   NODE=$(_log_docker_node "${TASK_NODE}");
-  MESSAGE=$(echo "${*}" | cut -d '|' -f 2-);
+  MESSAGE=$(extract_string "${*}" '|' 2-);
   # Remove the leading space.
-  SPACE=$(echo "${MESSAGE} " | cut -d ' ' -f 1)
-  [ -z "${SPACE}" ] && MESSAGE=$(echo "${MESSAGE} " | cut -d ' ' -f 2-)
-  FIRST_WORD=$(echo "${MESSAGE} " | cut -d ' ' -f 1);
+  SPACE=$(extract_string "${MESSAGE}" ' ' 1)
+  [ -z "${SPACE}" ] && MESSAGE=$(extract_string "${MESSAGE}" ' ' 2-)
+  FIRST_WORD=$(extract_string "${MESSAGE}" ' ' 1);
   if _log_level "${FIRST_WORD}" >/dev/null; then
     LEVEL=${FIRST_WORD};
-    MESSAGE=$(echo "${MESSAGE} " | cut -d ' ' -f 2-);
+    MESSAGE=$(extract_string "${MESSAGE}" ' ' 2-);
   fi
   _log_formatter "${LEVEL}" "${TIME}" "${NODE}" "${SCOPE}" "${MESSAGE}";
 }
@@ -182,7 +206,7 @@ is_number() {
 
 is_true() {
   local CONFIG="${1}"
-  CONFIG=$(echo "${CONFIG} " | cut -d ' ' -f 1)
+  CONFIG=$(extract_string "${CONFIG}" ' ' 1)
   echo "${CONFIG}" | grep_q_i "true"
 }
 
@@ -411,7 +435,7 @@ wait_service_state() {
         local TASK_RETURN_VALUE=
         TASK_RETURN_VALUE=$(echo "${STATES}" | grep "Failed" | sed -n 's/.*task: non-zero exit (\([0-9]\+\)).*/\1/p')
         # Get the first error code.
-        RETURN_VALUE=$(echo "${TASK_RETURN_VALUE:-1} " | cut -d ' ' -f 1)
+        RETURN_VALUE=$(extract_string "${TASK_RETURN_VALUE:-1}" ' ' 1)
         DOCKER_CMD_ERROR=0
         break
       fi
@@ -464,8 +488,8 @@ docker_current_container_name() {
       # '<container name>/<ip>/<mask>'
       # '<container name>/' (when network mode is host)
       local CNAME CIP
-      CNAME=$(echo "${NAME_AND_IP}/" | cut -d/ -f1);
-      CIP=$(echo "${NAME_AND_IP}/" | cut -d/ -f2);
+      CNAME=$(extract_string "${NAME_AND_IP}" '/' 1);
+      CIP=$(extract_string "${NAME_AND_IP}" '/' 2);
       # Unable to find the container IP when network mode is host.
       [ -z "${CIP}" ] && continue;
       for IP in ${IPS}; do
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index 10a3055..4a37a67 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -190,11 +190,11 @@ _authenticate_to_registries() {
     [ "${LINE:0:1}" = "#" ] && continue
     LINE=$(echo "${LINE}" | tr '\t' ' ')
     local OTHERS=
-    CONFIG=$(echo "${LINE} " | cut -d ' ' -f 1)
-    HOST=$(echo "${LINE} " | cut -d ' ' -f 2)
-    USER=$(echo "${LINE} " | cut -d ' ' -f 3)
-    PASSWORD=$(echo "${LINE} " | cut -d ' ' -f 4)
-    OTHERS=$(echo "${LINE} " | cut -d ' ' -f 5-)
+    CONFIG=$(extract_string "${LINE}" ' ' 1)
+    HOST=$(extract_string "${LINE}" ' ' 2)
+    USER=$(extract_string "${LINE}" ' ' 3)
+    PASSWORD=$(extract_string "${LINE}" ' ' 4)
+    OTHERS=$(extract_string "${LINE}" ' ' 5-)
     local ERROR_MSG=
     if [ -n "${OTHERS}" ]; then
       ERROR_MSG="Found extra item(s)."
@@ -763,10 +763,8 @@ _inspect_image() {
   fi
   local IMAGE=
   local DIGEST=
-  # If IMAGE_WITH_DIGEST contains no "@", then "cut -d@ -f2" will also return the entire string.
-  # Adding a "@" to ensure the string contains at least one "@". Thus DIGEST will be empty when original IMAGE_WITH_DIGEST contains no "@"
-  IMAGE=$(echo "${IMAGE_WITH_DIGEST}@" | cut -d@ -f1)
-  DIGEST=$(echo "${IMAGE_WITH_DIGEST}@" | cut -d@ -f2)
+  IMAGE=$(extract_string "${IMAGE_WITH_DIGEST}" '@' 1)
+  DIGEST=$(extract_string "${IMAGE_WITH_DIGEST}" '@' 2)
   if echo "${MANIFEST_CMD}" | grep_q_i "none"; then
     if _service_is_self "${SERVICE_NAME}"; then
       # Always inspecting self, never skipping.
@@ -859,7 +857,7 @@ _get_number_of_running_tasks() {
   # The REPLICAS is like "5/5" or "1/1 (3/5 completed)"
   # Get the number before the first "/".
   local NUM_RUNS=
-  NUM_RUNS=$(echo "${REPLICAS}/" | cut -d '/' -f 1)
+  NUM_RUNS=$(extract_string "${REPLICAS}" '/' 1)
   echo "${NUM_RUNS}"
 }
 

From eda12b04392c365327fb2cbd94bc1f0cfa17ad96 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Mon, 18 Nov 2024 12:19:26 -0800
Subject: [PATCH 02/27] [gantry] small refactor _report_services. Change local
 var name to avoid conflict with global.

---
 src/lib-gantry.sh           | 82 +++++++++++++++++++------------------
 tests/gantry_notify_spec.sh |  6 ++-
 2 files changed, 46 insertions(+), 42 deletions(-)

diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index 4a37a67..c85d956 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -108,17 +108,17 @@ _login_registry() {
    log WARN "HOST is empty. Will login to the default registry."
    REGISTRY_MESSAGE="default registry"
   fi
-  local DOCKER_CONFIG=
+  local AUTH_CONFIG=
   local CONFIG_MESSAGE="with default configuration"
   if [ -n "${CONFIG}" ]; then
-    DOCKER_CONFIG="--config ${CONFIG}"
+    AUTH_CONFIG="--config ${CONFIG}"
     CONFIG_MESSAGE="with configuration ${CONFIG}"
   fi
   local REGISTRY_CONFIG_MESSAGE="${REGISTRY_MESSAGE} ${CONFIG_MESSAGE}"
   local LOGIN_MSG=
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! LOGIN_MSG=$(echo "${PASSWORD}" | docker ${DOCKER_CONFIG} login --username="${USER}" --password-stdin "${HOST}" 2>&1); then
+  if ! LOGIN_MSG=$(echo "${PASSWORD}" | docker ${AUTH_CONFIG} login --username="${USER}" --password-stdin "${HOST}" 2>&1); then
     log ERROR "Failed to login to ${REGISTRY_CONFIG_MESSAGE}. ${LOGIN_MSG}"
     return 1
   fi
@@ -523,28 +523,30 @@ _report_services() {
   if [ "${NUM_FAILED}" = "0" ] && [ "${NUM_ERRORS}" = "0" ]; then
     NUM_ERRORS="${ACCUMULATED_ERRORS}"
   fi
-  local NUM_TOTAL_ERRORS=$((NUM_FAILED+NUM_ERRORS))
-  local TYPE="success"
-  [ "${NUM_TOTAL_ERRORS}" != "0" ] && TYPE="failure"
-  local ERROR_STRING=
-  [ "${NUM_ERRORS}" != "0" ] && ERROR_STRING=" ${NUM_TOTAL_ERRORS} error(s)"
-  local TITLE BODY SEND_NOTIFICATION
-  TITLE="[${STACK}] ${NUM_UPDATED} services updated ${NUM_FAILED} failed${ERROR_STRING}"
-  BODY=$(echo -e "${UPDATED_MSG}\n${FAILED_MSG}\n${ERROR_MSG}")
-  SEND_NOTIFICATION="true"
+  local NUM_FAILED_PLUS_ERRORS=$((NUM_FAILED+NUM_ERRORS))
+  local SEND_NOTIFICATION="true"
   case "${CONDITION}" in
     "on-change")
-      if [ "${NUM_UPDATED}" = "0" ] && [ "${NUM_TOTAL_ERRORS}" = "0" ]; then
-        log DEBUG "Skip sending notification because there are no updates or errors."
+      if [ "${NUM_UPDATED}" = "0" ] && [ "${NUM_FAILED_PLUS_ERRORS}" = "0" ]; then
+        log DEBUG "There are no updates or errors for notification."
         SEND_NOTIFICATION="false"
       fi
       ;;
     "all"|*)
       ;;
   esac
-  if is_true "${SEND_NOTIFICATION}"; then
-    _send_notification "${TYPE}" "${TITLE}" "${BODY}"
+  if ! is_true "${SEND_NOTIFICATION}"; then
+    log DEBUG "Skip sending notification."
+    return 0
   fi
+  local TYPE="success"
+  [ "${NUM_FAILED_PLUS_ERRORS}" != "0" ] && TYPE="failure"
+  local ERROR_STRING=
+  [ "${NUM_ERRORS}" != "0" ] && ERROR_STRING=" ${NUM_ERRORS} error(s)"
+  local TITLE BODY
+  TITLE="[${STACK}] ${NUM_UPDATED} services updated ${NUM_FAILED} failed${ERROR_STRING}"
+  BODY=$(echo -e "${UPDATED_MSG}\n${FAILED_MSG}\n${ERROR_MSG}")
+  _send_notification "${TYPE}" "${TITLE}" "${BODY}"
 }
 
 # Return 0 if the item is in the list.
@@ -670,9 +672,9 @@ _service_is_replicated() {
 
 # Return 0 if AUTH_CONFIG is a directory that contains Docker configuration files
 # Return 1 if AUTH_CONFIG is not a directory that contains Docker configuration files
-_check_docker_config_folder() {
+_check_auth_config_folder() {
   local AUTH_CONFIG="${1}"
-  # This is not a complete check.
+  # We only check whether it is a folder, thus it is not a complete check whether the folder contains valid Docker configuration files.
   if [ -d "${AUTH_CONFIG}" ]; then
     return 0
   fi
@@ -685,13 +687,13 @@ _check_docker_config_folder() {
   return 1
 }
 
-_get_config_from_service() {
+_get_auth_config_from_service() {
   local SERVICE_NAME="${1}"
   local AUTH_CONFIG_LABEL="gantry.auth.config"
   local AUTH_CONFIG=
   AUTH_CONFIG=$(_get_label_from_service "${SERVICE_NAME}" "${AUTH_CONFIG_LABEL}")
   [ -z "${AUTH_CONFIG}" ] && return 0
-  _check_docker_config_folder "${AUTH_CONFIG}"
+  _check_auth_config_folder "${AUTH_CONFIG}"
   echo "--config ${AUTH_CONFIG}"
 }
 
@@ -716,9 +718,9 @@ _get_image_info() {
   MANIFEST_OPTIONS=$(_read_env_or_label "${SERVICE_NAME}" "GANTRY_MANIFEST_OPTIONS" "gantry.manifest.options" "")
   local MANIFEST_CMD="${2}"
   local IMAGE="${3}"
-  local DOCKER_CONFIG=
-  DOCKER_CONFIG=$(_get_config_from_service "${SERVICE_NAME}")
-  [ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands for ${SERVICE_NAME}."
+  local AUTH_CONFIG=
+  AUTH_CONFIG=$(_get_auth_config_from_service "${SERVICE_NAME}")
+  [ -n "${AUTH_CONFIG}" ] && log DEBUG "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
   local MSG=
   local RETURN_VALUE=0
   if echo "${MANIFEST_CMD}" | grep_q_i "buildx"; then
@@ -726,13 +728,13 @@ _get_image_info() {
     [ -n "${MANIFEST_OPTIONS}" ] && log DEBUG "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker buildx imagetools inspect\"."
     # SC2086: Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2086
-    MSG=$(docker ${DOCKER_CONFIG} buildx imagetools inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
+    MSG=$(docker ${AUTH_CONFIG} buildx imagetools inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
     RETURN_VALUE=$?
   elif echo "${MANIFEST_CMD}" | grep_q_i "manifest"; then
     [ -n "${MANIFEST_OPTIONS}" ] && log DEBUG "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker manifest inspect\"."
     # SC2086: Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2086
-    MSG=$(docker ${DOCKER_CONFIG} manifest inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
+    MSG=$(docker ${AUTH_CONFIG} manifest inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
     RETURN_VALUE=$?
   elif echo "${MANIFEST_CMD}" | grep_q_i "none"; then
     # We should never reach here, the "none" command is already checked inside the function _inspect_image.
@@ -862,12 +864,12 @@ _get_number_of_running_tasks() {
 }
 
 _get_with_registry_auth() {
-  local DOCKER_CONFIG="${1}"
+  local AUTH_CONFIG="${1}"
   local ARGUMENTS=""
-  # DOCKER_CONFIG is currently only used by Authentication.
+  # AUTH_CONFIG is currently (2024.11) only used by Authentication.
   # When login is required, we must add `--with-registry-auth`. Otherwise the service will get an image without digest.
   # See https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336
-  if [ -n "${DOCKER_CONFIG}" ] || _docker_default_config_is_used; then
+  if [ -n "${AUTH_CONFIG}" ] || _docker_default_config_is_used; then
     ARGUMENTS="--with-registry-auth";
   fi
   echo "${ARGUMENTS}"
@@ -875,7 +877,7 @@ _get_with_registry_auth() {
 
 _get_service_update_additional_options() {
   local SERVICE_NAME="${1}"
-  local DOCKER_CONFIG="${2}"
+  local AUTH_CONFIG="${2}"
   local NUM_RUNS=
   NUM_RUNS=$(_get_number_of_running_tasks "${SERVICE_NAME}")
   if ! is_number "${NUM_RUNS}"; then
@@ -895,14 +897,14 @@ _get_service_update_additional_options() {
   fi
   # Add `--with-registry-auth` if needed.
   local WITH_REGISTRY_AUTH=
-  WITH_REGISTRY_AUTH="$(_get_with_registry_auth "${DOCKER_CONFIG}")"
+  WITH_REGISTRY_AUTH="$(_get_with_registry_auth "${AUTH_CONFIG}")"
   [ -n "${WITH_REGISTRY_AUTH}" ] && OPTIONS="${OPTIONS} ${WITH_REGISTRY_AUTH}"
   echo "${OPTIONS}"
 }
 
 _get_service_rollback_additional_options() {
   local SERVICE_NAME="${1}"
-  local DOCKER_CONFIG="${2}"
+  local AUTH_CONFIG="${2}"
   local OPTIONS=
   # Place holder function. Nothing to do here yet.
   # --with-registry-auth cannot be combined with --rollback.
@@ -915,21 +917,21 @@ _rollback_service() {
   ROLLBACK_ON_FAILURE=$(_read_env_or_label "${SERVICE_NAME}" "GANTRY_ROLLBACK_ON_FAILURE" "gantry.rollback.on_failure" "true")
   local ROLLBACK_OPTIONS=
   ROLLBACK_OPTIONS=$(_read_env_or_label "${SERVICE_NAME}" "GANTRY_ROLLBACK_OPTIONS" "gantry.rollback.options" "")
-  local DOCKER_CONFIG="${2}"
+  local AUTH_CONFIG="${2}"
   if ! is_true "${ROLLBACK_ON_FAILURE}"; then
     return 0
   fi
   log INFO "Rolling back ${SERVICE_NAME}."
   # "service update --rollback" needs to take different options from "service update"
   local ADDITIONAL_OPTIONS=
-  ADDITIONAL_OPTIONS=$(_get_service_rollback_additional_options "${SERVICE_NAME}" "${DOCKER_CONFIG}")
+  ADDITIONAL_OPTIONS=$(_get_service_rollback_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
   [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
   [ -n "${ROLLBACK_OPTIONS}" ] && log DEBUG "Adding options \"${ROLLBACK_OPTIONS}\" to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
   local ROLLBACK_MSG=
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! ROLLBACK_MSG=$(docker ${DOCKER_CONFIG} service update --quiet ${ADDITIONAL_OPTIONS} ${ROLLBACK_OPTIONS} --rollback "${SERVICE_NAME}" 2>&1); then
+  if ! ROLLBACK_MSG=$(docker ${AUTH_CONFIG} service update --quiet ${ADDITIONAL_OPTIONS} ${ROLLBACK_OPTIONS} --rollback "${SERVICE_NAME}" 2>&1); then
     log ERROR "Failed to roll back ${SERVICE_NAME}. ${ROLLBACK_MSG}"
     return 1
   fi
@@ -974,16 +976,16 @@ _update_single_service() {
   local START_TIME=
   START_TIME=$(date +%s)
   log INFO "Updating ${SERVICE_NAME} with image ${IMAGE}"
-  local DOCKER_CONFIG=
+  local AUTH_CONFIG=
   local ADDITIONAL_OPTIONS=
-  DOCKER_CONFIG=$(_get_config_from_service "${SERVICE_NAME}")
-  ADDITIONAL_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}" "${DOCKER_CONFIG}")
-  [ -n "${DOCKER_CONFIG}" ] && log DEBUG "Adding options \"${DOCKER_CONFIG}\" to docker commands for ${SERVICE_NAME}."
+  AUTH_CONFIG=$(_get_auth_config_from_service "${SERVICE_NAME}")
+  ADDITIONAL_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
+  [ -n "${AUTH_CONFIG}" ] && log DEBUG "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
   [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update\" for ${SERVICE_NAME}."
   [ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" to the command \"docker service update\" for ${SERVICE_NAME}."
   local TIMEOUT_COMMAND=""
   TIMEOUT_COMMAND=$(_get_timeout_command "${SERVICE_NAME}") || return 1
-  local UPDATE_COMMAND="${TIMEOUT_COMMAND} docker ${DOCKER_CONFIG} service update"
+  local UPDATE_COMMAND="${TIMEOUT_COMMAND} docker ${AUTH_CONFIG} service update"
   local UPDATE_RETURN_VALUE=0
   local UPDATE_MSG=
   # Add "-quiet" to suppress progress output.
@@ -1000,7 +1002,7 @@ _update_single_service() {
     fi
     log ERROR "Command \"${UPDATE_COMMAND}\" returns ${UPDATE_RETURN_VALUE}. ${TIMEOUT_MSG}"
     log ERROR "docker service update failed. ${UPDATE_MSG}"
-    _rollback_service "${SERVICE_NAME}" "${DOCKER_CONFIG}"
+    _rollback_service "${SERVICE_NAME}" "${AUTH_CONFIG}"
     _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_FAILED "${SERVICE_NAME}"
     return 1
   fi
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index f2d58c8..6ddd76b 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -18,7 +18,8 @@
 export TOTAL_EMAIL_COUNT_IS_ONE="\"total\": *1,"
 export SEND_NOTIFY_APPRISE="Sent notification via Apprise"
 export SKIP_NOTIFY_APPRISE="Skip sending notification via Apprise"
-export SKIP_NO_UPDATES="Skip sending notification because there are no updates or errors"
+export NO_UPDATES_OR_ERRORS_FOR_NOTIFICATION="There are no updates or errors for notification."
+export SKIP_SENDING_NOTIFICATION="Skip sending notification."
 
 UINQUE_ID="$(unique_id)"
 export SERVICE_NAME_APPRISE="gantry-test-${UINQUE_ID}-apprise"
@@ -320,7 +321,8 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_message    "${SKIP_NO_UPDATES}"
+      The stderr should satisfy spec_expect_message    "${NO_UPDATES_OR_ERRORS_FOR_NOTIFICATION}"
+      The stderr should satisfy spec_expect_message    "${SKIP_SENDING_NOTIFICATION}"
     End
   End
   Describe "test_notify_on_change_errors" "container_test:false"

From 842f83f2871864c7a8b85b1071eb1dd41e7a6f46 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Mon, 18 Nov 2024 22:18:42 -0800
Subject: [PATCH 03/27] [tests] Add test_common_no_new_env. Print the loading
 lib message when log level is debug.

---
 src/entrypoint.sh                   | 17 ++++++++----
 src/lib-common.sh                   |  5 ++++
 src/lib-gantry.sh                   |  1 +
 tests/gantry_common_options_spec.sh | 43 +++++++++++++++++++++++++++++
 4 files changed, 60 insertions(+), 6 deletions(-)

diff --git a/src/entrypoint.sh b/src/entrypoint.sh
index 8c78a6b..38e3b11 100755
--- a/src/entrypoint.sh
+++ b/src/entrypoint.sh
@@ -28,7 +28,7 @@ _get_lib_dir() {
   elif [ -r "./lib-gantry.sh" ]; then
     LIB_DIR="."
   fi
-  echo "${LIB_DIR}"
+  readlink -f "${LIB_DIR}"
 }
 
 _log_load_libraries() {
@@ -36,14 +36,19 @@ _log_load_libraries() {
   local IMAGES_TO_REMOVE="${GANTRY_IMAGES_TO_REMOVE:-""}"
   local LIB_DIR="${1}"
   # log function is not available before loading the library.
-  if echo "${LOG_LEVEL}" | grep -q -i "NONE"; then
+  local LOADING_MSG="Loading libraries from ${LIB_DIR}"
+  if [ -n "${IMAGES_TO_REMOVE}" ]; then
+    echo "DEBUG ${LOADING_MSG}" >&2
+    return 0;
+  fi
+  # DEBUG should be the lowest level.
+  if ! echo "${LOG_LEVEL}" | grep -q -i "^DEBUG$"; then
     return 0
   fi
   local TIMESTAMP=
-  if [ -z "${IMAGES_TO_REMOVE}" ]; then
-    TIMESTAMP="[$(date -Iseconds)] "
-  fi
-  echo "${TIMESTAMP}Loading libraries from ${LIB_DIR}" >&2
+  TIMESTAMP="[$(date -Iseconds)]"
+  local LEVEL="[DEBUG]"
+  echo "${TIMESTAMP}${LEVEL} ${LOADING_MSG}" >&2
 }
 
 load_libraries() {
diff --git a/src/lib-common.sh b/src/lib-common.sh
index cd36447..813f9b0 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -194,6 +194,7 @@ _log_docker_line() {
 # Usage: echo "${LOGS}" | log_lines INFO
 log_lines() {
   local LEVEL="${1}";
+  local LINE=;
   while read -r LINE; do
     [ -z "${LINE}" ] && continue;
     log "${LEVEL}" "${LINE}";
@@ -412,6 +413,7 @@ wait_service_state() {
     local NUM_RUNS=0
     local NUM_DONES=0
     local NUM_FAILS=0
+    local LINE=
     while read -r LINE; do
       [ -z "${LINE}" ] && continue;
       NUM_LINES=$((NUM_LINES+1));
@@ -446,6 +448,7 @@ wait_service_state() {
     log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${STATES}"
     return 1
   fi
+  local LINE=
   while read -r LINE; do
     log INFO "Service ${SERVICE_NAME}: ${LINE}."
   done < <(echo "${STATES}")
@@ -482,6 +485,7 @@ docker_current_container_name() {
     [ "${NID}" = "${HOST_NETWORK}" ] && continue;
     local ALL_LOCAL_NAME_AND_IP=;
     ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1;
+    local NAME_AND_IP=;
     for NAME_AND_IP in ${ALL_LOCAL_NAME_AND_IP}; do
       [ -z "${NAME_AND_IP}" ] && continue;
       # NAME_AND_IP will be in one of the following formats:
@@ -492,6 +496,7 @@ docker_current_container_name() {
       CIP=$(extract_string "${NAME_AND_IP}" '/' 2);
       # Unable to find the container IP when network mode is host.
       [ -z "${CIP}" ] && continue;
+      local IP=;
       for IP in ${IPS}; do
         [ "${IP}" != "${CIP}" ] && continue;
         echo "${CNAME}";
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index c85d956..341bacb 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -256,6 +256,7 @@ _remove_static_variables_folder() {
   TO_REMOVE_STATIC_VARIABLES_FOLDER="$(_get_static_variables_folder)"
   log DEBUG "Removing STATIC_VARIABLES_FOLDER ${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
   export STATIC_VARIABLES_FOLDER=
+  unset STATIC_VARIABLES_FOLDER
   rm -r "${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
 }
 
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index fad512c..f64bd55 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -91,6 +91,49 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message ".+"
     End
   End
+  Describe "test_common_no_new_env" "container_test:false"
+    # Check there is no new variable set,
+    # to avoid errors like https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
+    # We don't need to run this test using containers because we check env on the host, while the container test set env inside the container.
+    TEST_NAME="test_common_no_new_env"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    test_common_no_new_env() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      local ENV_BEFORE_RUN=
+      ENV_BEFORE_RUN=$(mktemp)
+      local ENV_AFTER_RUN=
+      ENV_AFTER_RUN=$(mktemp)
+
+      reset_gantry_env "${SERVICE_NAME}"
+      # There should be no warnings or errors. So it should work the same as LOG_LEVLE=NONE.
+      export GANTRY_LOG_LEVEL=WARN
+
+      # _ contains the last command. declare is a bash builtin.
+      unset _; declare -p > "${ENV_BEFORE_RUN}"
+      run_gantry "${TEST_NAME}"
+
+      # Allow the following 3 mismatches used in log() function.
+      unset LOG_LEVEL
+      unset NODE_NAME
+      unset LOG_SCOPE
+      unset _; declare -p > "${ENV_AFTER_RUN}"
+      diff <(cat "${ENV_BEFORE_RUN}") <(cat "${ENV_AFTER_RUN}")
+      rm "${ENV_BEFORE_RUN}"
+      rm "${ENV_AFTER_RUN}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_common_no_new_env "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message ".+"
+    End
+  End
   Describe "test_common_PRE_POST_RUN_CMD" "container_test:true"
     TEST_NAME="test_common_PRE_POST_RUN_CMD"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")

From a8daa323b3e7b19f5f8779214dd585fd2499b6b3 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 19 Nov 2024 16:26:07 -0800
Subject: [PATCH 04/27] [docs] Update Authentication, refine
 GANTRY_MANIFEST_CMD.

---
 README.md              | 66 ++++++++++++++----------------------------
 docs/authentication.md | 63 ++++++++++++++++++++++++++++++++++++++++
 docs/faq.md            | 16 ++++++----
 docs/migration.md      | 16 ++++++----
 4 files changed, 106 insertions(+), 55 deletions(-)
 create mode 100644 docs/authentication.md

diff --git a/README.md b/README.md
index f7af711..7049251 100644
--- a/README.md
+++ b/README.md
@@ -53,15 +53,16 @@ You can configure the most behaviors of *Gantry* via environment variables.
 
 | Environment Variable  | Default | Description |
 |-----------------------|---------|-------------|
-| GANTRY_REGISTRY_CONFIG        | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_CONFIG_FILE   | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_CONFIGS_FILE  | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_HOST          | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_HOST_FILE     | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_PASSWORD      | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_PASSWORD_FILE | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_USER          | | See [Authentication](#authentication). |
-| GANTRY_REGISTRY_USER_FILE     | | See [Authentication](#authentication). |
+| DOCKER_CONFIG                 | | The location of the [client configuration files](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files) where authentication stores. It applys to all Docker commands, i.e. to all services. See [Authentication](docs/authentication.md). You can apply a different value to a particular service via [labels](#labels). |
+| GANTRY_REGISTRY_CONFIG        | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_CONFIG_FILE   | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_CONFIGS_FILE  | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_HOST          | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_HOST_FILE     | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_PASSWORD      | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_PASSWORD_FILE | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_USER          | | See [Authentication](docs/authentication.md). |
+| GANTRY_REGISTRY_USER_FILE     | | See [Authentication](docs/authentication.md). |
 
 ### To select services
 
@@ -75,7 +76,7 @@ You can configure the most behaviors of *Gantry* via environment variables.
 
 | Environment Variable  | Default | Description |
 |-----------------------|---------|-------------|
-| GANTRY_MANIFEST_CMD         | buildx | Valid values are `buildx`, `manifest`, and `none`.<br>Set which command for manifest inspection. Also see FAQ section [when to set `GANTRY_MANIFEST_CMD`](docs/faq.md#when-to-set-gantry_manifest_cmd).<ul><li>[`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/)</li><li>[`docker manifest inspect`](https://docs.docker.com/engine/reference/commandline/manifest_inspect/)</li></ul>Set to `none` to skip checking the manifest. As a result of skipping, `docker service update` always runs. In case you add `--force` to `GANTRY_UPDATE_OPTIONS`, you also want to disable the inspection. You can apply a different value to a particular service via [labels](#labels). |
+| GANTRY_MANIFEST_CMD         | buildx | Valid values are `buildx`, `manifest`, and `none`.<br>Set which command for manifest inspection.<ul><li>[`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/)</li><li>[`docker manifest inspect`](https://docs.docker.com/engine/reference/commandline/manifest_inspect/)</li></ul>Set to `none` to skip checking the manifest. As a result of skipping, `docker service update` always runs. Also see FAQ [which `GANTRY_MANIFEST_CMD` to use](docs/faq.md#which-gantry_manifest_cmd-to-use). You can apply a different value to a particular service via [labels](#labels). |
 | GANTRY_MANIFEST_NUM_WORKERS | 1      | The maximum number of `GANTRY_MANIFEST_CMD` that can run in parallel. |
 | GANTRY_MANIFEST_OPTIONS     |        | [Options](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/#options) added to the `docker buildx imagetools inspect` or [options](https://docs.docker.com/engine/reference/commandline/manifest_inspect/#options) to `docker manifest inspect`, depending on `GANTRY_MANIFEST_CMD` value, for all services. You can apply a different value to a particular service via [labels](#labels). |
 
@@ -85,7 +86,7 @@ You can configure the most behaviors of *Gantry* via environment variables.
 |-----------------------|---------|-------------|
 | GANTRY_ROLLBACK_ON_FAILURE    | true  | Set to `true` to enable rollback when updating fails. Set to `false` to disable the rollback. You can apply a different value to a particular service via [labels](#labels). |
 | GANTRY_ROLLBACK_OPTIONS       |       | [Options](https://docs.docker.com/engine/reference/commandline/service_update/#options) added to the `docker service update --rollback` command for all services. You can apply a different value to a particular service via [labels](#labels). |
-| GANTRY_UPDATE_JOBS            | false | Set to `true` to update replicated-job or global-job. Set to `false` to disable updating jobs. *Gantry* adds additional options to `docker service update` when there is [no running tasks](docs/faq.md#how-to-update-services-with-no-running-tasks). You can apply a different value to a particular service via [labels](#labels). |
+| GANTRY_UPDATE_JOBS            | false | Set to `true` to update `replicated-job` or `global-job`. Set to `false` to disable updating jobs. *Gantry* adds additional options to `docker service update` when there is [no running tasks](docs/faq.md#how-to-update-services-with-no-running-tasks). You can apply a different value to a particular service via [labels](#labels). |
 | GANTRY_UPDATE_NUM_WORKERS     | 1     | The maximum number of updates that can run in parallel. |
 | GANTRY_UPDATE_OPTIONS         |       | [Options](https://docs.docker.com/engine/reference/commandline/service_update/#options) added to the `docker service update` command for all services. You can apply a different value to a particular service via [labels](#labels). |
 | GANTRY_UPDATE_TIMEOUT_SECONDS | 0     | Error out if updating of a single service takes longer than the given time. Set to `0` to disable timeout. You can apply a different value to a particular service via [labels](#labels). |
@@ -100,49 +101,26 @@ You can configure the most behaviors of *Gantry* via environment variables.
 | GANTRY_NOTIFICATION_CONDITION   | all   | Valid values are `all` and `on-change`. Specifies the conditions under which notifications are sent. Set to `all` to send notifications every run. Set to `on-change` to send notifications only when there are updates or errors. |
 | GANTRY_NOTIFICATION_TITLE       |       | Add an additional message to the notification title. |
 
-## Authentication
-
-If you only need to login to a single registry, you can use the environment variables  `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD`, `GANTRY_REGISTRY_HOST` and `GANTRY_REGISTRY_CONFIG` to provide the authentication information. You may also use the `*_FILE` variants to pass the information through files. The files can be added to the service via [docker secret](https://docs.docker.com/engine/swarm/secrets/). `GANTRY_REGISTRY_HOST` and `GANTRY_REGISTRY_CONFIG` are optional. Use `GANTRY_REGISTRY_HOST` when you are not using Docker Hub. Use `GANTRY_REGISTRY_CONFIG` when you want to enable authentication for only selected services.
-
-If the images of services are hosted on multiple registries that are required authentication, you should provide a configuration file to the *Gantry* and set `GANTRY_REGISTRY_CONFIGS_FILE` correspondingly. You can use [docker secret](https://docs.docker.com/engine/swarm/secrets/) to provision the configuration file. The configuration file must be in the following format:
-
-* Each line should contain 4 columns, which are either `<TAB>` or `<SPACE>` separated. The columns are 
-```
-<configuration> <host> <user> <password>
-```
-> * configuration: an identifier for the account. This is used as a path to [Docker configuration files](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files), which could be either a relative path or an absolute path.
-> * host: the registry to authenticate against, e.g. docker.io.
-> * user: the user name to authenticate as.
-> * password: the password to authenticate with.
-* Lines starting with  `#` are comments.
-* Empty lines, comment lines and invalid lines are ignored.
-
-You need to tell *Gantry* to use a named configuration rather than the default one when updating a particular service. The named configurations are set via either `GANTRY_REGISTRY_CONFIG`, `GANTRY_REGISTRY_CONFIG_FILE` or `GANTRY_REGISTRY_CONFIGS_FILE`. This can be done by adding the following label to the service `gantry.auth.config=<configuration>`. *Gantry* creates [Docker configuration files](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files) and adds `--config <configuration>` to the Docker command line for the corresponding services.
-
-> NOTE: *Gantry* automatically adds `--with-registry-auth` to the `docker service update` command for a service, when it finds the label `gantry.auth.config=<configuration>` on the service, or when it logs in with the default Docker configuration. Without `--with-registry-auth`, the service will be updated to an image without digest. See this [comment](https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336).
-
-> NOTE: You can use `GANTRY_REGISTRY_CONFIGS_FILE` together with other authentication environment variables.
-
-> NOTE: *Gantry* uses `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER` to obtain Docker Hub rate when `GANTRY_REGISTRY_HOST` is empty or `docker.io`. You can also use their `_FILE` variants. If either password or user is empty, *Gantry* reads the Docker Hub rate for anonymous users.
-
 ## Labels
 
 Labels can be added to services to modify the behavior of *Gantry* for particular services. When *Gantry* sees the following labels on a service, it will modify the Docker command line only for that service. The value on the label overrides the global environment variables.
 
 | Label  | Description |
 |--------|-------------|
-| `gantry.auth.config=<configuration>`     | See [Authentication](#authentication). |
+| `gantry.auth.config=<configuration>`     | Override [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables). See [Authentication](docs/authentication.md). |
 | `gantry.services.excluded=true`          | Exclude the services from updating if you are using the default [`GANTRY_SERVICES_EXCLUDED_FILTERS`](#to-select-services). |
-| `gantry.manifest.cmd=<command>`          | Override [`GANTRY_MANIFEST_CMD`](#to-check-if-new-images-are-available) |
-| `gantry.manifest.options=<string> `      | Override [`GANTRY_MANIFEST_OPTIONS`](#to-check-if-new-images-are-available) |
-| `gantry.rollback.on_failure=<boolean>`   | Override [`GANTRY_ROLLBACK_ON_FAILURE`](#to-add-options-to-services-update) |
-| `gantry.rollback.options=<string>`       | Override [`GANTRY_ROLLBACK_OPTIONS`](#to-add-options-to-services-update) |
-| `gantry.update.jobs=<boolean>`           | Override [`GANTRY_UPDATE_JOBS`](#to-add-options-to-services-update) |
-| `gantry.update.options=<string>`         | Override [`GANTRY_UPDATE_OPTIONS`](#to-add-options-to-services-update) |
-| `gantry.update.timeout_seconds=<number>` | Override [`GANTRY_UPDATE_TIMEOUT_SECONDS`](#to-add-options-to-services-update) |
+| `gantry.manifest.cmd=<command>`          | Override [`GANTRY_MANIFEST_CMD`](#to-check-if-new-images-are-available). |
+| `gantry.manifest.options=<string> `      | Override [`GANTRY_MANIFEST_OPTIONS`](#to-check-if-new-images-are-available). |
+| `gantry.rollback.on_failure=<boolean>`   | Override [`GANTRY_ROLLBACK_ON_FAILURE`](#to-add-options-to-services-update). |
+| `gantry.rollback.options=<string>`       | Override [`GANTRY_ROLLBACK_OPTIONS`](#to-add-options-to-services-update). |
+| `gantry.update.jobs=<boolean>`           | Override [`GANTRY_UPDATE_JOBS`](#to-add-options-to-services-update). |
+| `gantry.update.options=<string>`         | Override [`GANTRY_UPDATE_OPTIONS`](#to-add-options-to-services-update). |
+| `gantry.update.timeout_seconds=<number>` | Override [`GANTRY_UPDATE_TIMEOUT_SECONDS`](#to-add-options-to-services-update). |
 
 ## FAQ
 
+[Authentication](docs/authentication.md)
+
 [FAQ](docs/faq.md)
 
 [Migrate from *Shepherd*](docs/migration.md)
diff --git a/docs/authentication.md b/docs/authentication.md
new file mode 100644
index 0000000..08e0073
--- /dev/null
+++ b/docs/authentication.md
@@ -0,0 +1,63 @@
+## Authentication
+
+### Single registry
+
+If you only need to login to a single registry, you can use the environment variables `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD`, `GANTRY_REGISTRY_HOST` and `GANTRY_REGISTRY_CONFIG` to provide the authentication information. You may also use the `*_FILE` variants to pass the information through files. The files can be added to the service via [Docker secret](https://docs.docker.com/engine/swarm/secrets/).
+
+`GANTRY_REGISTRY_HOST` is optional. Use `GANTRY_REGISTRY_HOST` when you are not using Docker Hub.
+
+`GANTRY_REGISTRY_CONFIG` is optional. Use `GANTRY_REGISTRY_CONFIG` when you want to enable authentication for only selected services. When `GANTRY_REGISTRY_CONFIG` is empty, *Gantry* login using the default [Docker configuration](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files). When `GANTRY_REGISTRY_CONFIG` is set, *Gantry* use it as the path of the Docker configuration folder.
+
+> NOTE: *Gantry* uses `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER` to obtain Docker Hub rate when `GANTRY_REGISTRY_HOST` is empty or `docker.io`. You can also use their `_FILE` variants. If either password or user is empty, *Gantry* reads the Docker Hub rate for anonymous users.
+
+### Multiple registries
+
+If the images of services are hosted on multiple registries that are required authentication, you should provide a configuration file to the *Gantry* and set `GANTRY_REGISTRY_CONFIGS_FILE` correspondingly. You can use [Docker secret](https://docs.docker.com/engine/swarm/secrets/) to provision the configuration file. The configuration file must be in the following format:
+
+* Each line should contain 4 columns, which are either `<TAB>` or `<SPACE>` separated. The columns are
+
+```
+<configuration> <host> <user> <password>
+```
+
+> * configuration: The location of the [client configuration files](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files) where authentication stores. It could be either a relative path or an absolute path.
+> * host: the registry to authenticate against, e.g. docker.io.
+> * user: the user name to authenticate as.
+> * password: the password to authenticate with.
+
+* Lines starting with  `#` are comments.
+* Empty lines, comment lines and invalid lines are ignored.
+
+You can use `GANTRY_REGISTRY_CONFIGS_FILE` together with other authentication environment variables.
+
+You can login to multiple registries using the same Docker configuration. However if you login to the same registry with different user names for different services, you should use different Docker configurations.
+
+### Select Docker configurations for services
+
+If you login to a single registry using `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_HOST` without setting `GANTRY_REGISTRY_CONFIG`, the default Docker configuration is used. You don't need to set anything extra for authentication.
+
+ *Gantry* creates or updates the docker configurations based on the locations set via `GANTRY_REGISTRY_CONFIG`, `GANTRY_REGISTRY_CONFIG_FILE` or `GANTRY_REGISTRY_CONFIGS_FILE`.
+
+ You can use environment variable [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables) to apply the same Docker configuration to all docker commands, i.e. to all services.
+
+You need to add label `gantry.auth.config=<configuration>` to particular services to tell which Docker configuration to use for authentication, when you use different configurations for different registries. When *Gantry* finds the label `gantry.auth.config=<configuration>` on services, it adds `--config <configuration>` to the Docker commands for the corresponding services.
+
+*Gantry* automatically adds `--with-registry-auth` to the `docker service update` command for services for the following cases.
+
+* when *Gantry* finds the label `gantry.auth.config=<configuration>` on the service.
+* when *Gantry* logs in with the default Docker configuration.
+  * when `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` are set, while `GANTRY_REGISTRY_CONFIG` is empty.
+  * when the configuration from `GANTRY_REGISTRY_CONFIG` or `GANTRY_REGISTRY_CONFIGS_FILE` is same as the default Docker configuration location `${HOME}/.docker/` or the location specified by `DOCKER_CONFIG`.
+
+You can manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` if it is not added automatically for your case. Without `--with-registry-auth`, the service will be [updated to an image without digest](https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336), and you might get a warning "*image \<image\> could not be accessed on a registry to record its digest. Each node will access \<image\> independently, possibly leading to different nodes running different versions of the image.*"
+
+### Use an existing Docker configuration
+
+You can use an existing Docker configuration from the host machines for authorization when you run *Gantry* as a Docker service. You need to do the followings.
+
+* Log into registries on the host using [`docker login`](https://docs.docker.com/reference/cli/docker/login/). The [default configuration](https://docs.docker.com/reference/cli/docker/#configuration-files) locates at `${HOME}/.docker/`. You can [change the `.docker` directory](https://docs.docker.com/reference/cli/docker/#change-the-docker-directory).
+* Mount the Docker configuration directory from the host to the container. You could just mount the `config.json` file using [Docker secret](https://docs.docker.com/engine/swarm/secrets/).
+* Set the environment variable `DOCKER_CONFIG` on the *Gantry* container to specify the location of the Docker configuration folder inside the container. You can skip this step when you mount the folder to the default Docker configuration location `/root/.docker/` inside the container.
+* Add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` manually.
+
+> Note that [`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/) writes data to the Docker configuration folder `${DOCKER_CONFIG}/buildx`, which therefore needs to be writable. You can set `GANTRY_MANIFEST_CMD` to `manifest` to avoid writing to the Docker configuration folder.
diff --git a/docs/faq.md b/docs/faq.md
index a720f37..303daad 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -28,18 +28,22 @@ As discussed [here](https://github.com/docker/cli/issues/627), the CLI will hang
 
 *Gantry* will check whether there are running tasks in a service. If there is no running task, *Gantry* automatically adds the option `--detach=true`. In addition to the detach option, *Gantry* also adds `--replicas=0` for services in replicated mode. You don't need to add these options manually.
 
-### What `GANTRY_MANIFEST_CMD` to use?
+### Which `GANTRY_MANIFEST_CMD` to use?
 
 Before updating a service, *Gantry* will try to obtain the image's meta data to decide whether there is a new image. If there is no new image, *Gantry* skips calling `docker service update`, leading to a speedup of the overall process.
 
-In most cases, the default value `buildx` of `GANTRY_MANIFEST_CMD` should work. `docker buildx imagetools inspect` is selected as the default, because `docker manifest inspect` could [fail on some registries](https://github.com/orgs/community/discussions/45779). Additionally, `docker buildx imagetools` can obtain the digest of multi-arch images, which could help reduce the number of calling the `docker service update` CLI when there is no new images.
+`buildx`, the default value, should work in most cases. [`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/) is selected as the default, because `docker manifest inspect` could [fail on some registries](https://github.com/orgs/community/discussions/45779). Additionally, `docker buildx imagetools` can obtain the digest of multi-arch images, which could help reduce the number of calling the `docker service update` CLI when there is no new images.
 
-We keep [`docker manifest inspect`](https://docs.docker.com/engine/reference/commandline/manifest_inspect/) for debugging purpose. There is no known advantage to use `manifest`.
+`manifest` is kept for debugging purpose. The only known advantage of [`docker manifest inspect`](https://docs.docker.com/engine/reference/commandline/manifest_inspect/) is that it does not require the write permission to the [Docker configuration folder](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files). If the Docker configuration folder is read-only, you need to use `manifest` to avoid permission deny errors.
 
-You can disable the image inspection by setting `GANTRY_MANIFEST_CMD` to `none` in case there is a bug. Please report the bug through a [GitHub issue](https://github.com/shizunge/gantry/issues). Another use case of `none` is that you want to add `--force` to the `docker service update` command via `GANTRY_UPDATE_OPTIONS`, which updates the services even if there is nothing changed.
+`none` can be used to disable the image inspection. One use case of `none` is that you want to add `--force` to the `docker service update` command via `GANTRY_UPDATE_OPTIONS`, which updates the services even if there is nothing changed. Another use case is to debug image inspection. Please report the bug through a [GitHub issue](https://github.com/shizunge/gantry/issues), thanks.
 
-### I logged in my Docker Hub account, but the Docker Hub rate reported seems incorrect.
+### Can *Gantry* report Docker Hub rate for non-anonymous account?
 
 When checking Docker Hub rate, *Gantry* reads the Docker Hub credential only from `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER`, or their `_FILE` variants. `GANTRY_REGISTRY_HOST` or its `_FILE` variant must be either empty or `docker.io`.
 
-If you need to login to multiple registries, you can use `GANTRY_REGISTRY_CONFIGS_FILE` together with `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER`. Credentials in `GANTRY_REGISTRY_CONFIGS_FILE` will be used for services updating, but they won't be used for checking Docker Hub rate. See [Authentication](../README.md#authentication) for more information.
+If you need to login to multiple registries, you can use `GANTRY_REGISTRY_CONFIGS_FILE` together with `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER`. Credentials in `GANTRY_REGISTRY_CONFIGS_FILE` will be used for services updating, but they won't be used for checking Docker Hub rate. See [Authentication](../docs/authentication.md) for more information.
+
+### How to use authorization from the host machines?
+
+See [use an existing Docker configuration](../docs/authentication.md#use-an-existing-docker-configuration).
diff --git a/docs/migration.md b/docs/migration.md
index 5e89805..f1af29d 100644
--- a/docs/migration.md
+++ b/docs/migration.md
@@ -6,8 +6,8 @@
 * High usage of Docker Hub rate. Getting manifest and then pulling the image double the usage.
 * Running `docker service update` command when there is no new image slows down the overall process.
 * Removing images related
-    * Failure of removing old images will exit and block subsequent updating.
-    * `docker rmi` only works for the current host.
+  * Failure of removing old images will exit and block subsequent updating.
+  * `docker rmi` only works for the current host.
 * `docker service update` CLI hangs when updating services without running tasks.
 * Other UX issues when running it as a script outside the provided container.
 
@@ -41,13 +41,19 @@ The label on the services to select config to enable authentication is renamed t
 
 | *Shepherd* Env | Workaround |
 |----------------|------------|
-| WITH_REGISTRY_AUTH     | *Gantry* automatically adds `--with-registry-auth` to the `docker service update` command for a sevice, when it finds the label `gantry.auth.config=<config-name>` on the service. Or manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS`. |
-| WITH_INSECURE_REGISTRY | Manually add `--insecure` to `GANTRY_MANIFEST_OPTIONS` and set `GANTRY_MANIFEST_CMD` to `manifest`. |
+| WITH_REGISTRY_AUTH     | *Gantry* automatically adds `--with-registry-auth` to the `docker service update` command for a sevice, when it finds the label `gantry.auth.config=<configuration>` on the service, or when it logs in with the default Docker configuration. You can also manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS`. |
+| WITH_INSECURE_REGISTRY | Manually add `--insecure` to `GANTRY_MANIFEST_OPTIONS` when `GANTRY_MANIFEST_CMD` is `manifest`. |
 | WITH_NO_RESOLVE_IMAGE  | Manually add `--no-resolve-image` to `GANTRY_UPDATE_OPTIONS`. |
 | RUN_ONCE_AND_EXIT      | Set `GANTRY_SLEEP_SECONDS` to 0. |
 
 ### New configurations
 
+You can enable these new features of *Gantry* through new configurations.
+
+* Execute pre-run and post-run commands via `GANTRY_PRE_RUN_CMD` and `GANTRY_POST_RUN_CMD`.
+* Exclude services via filters in addition to names via `GANTRY_SERVICES_EXCLUDED_FILTERS`.
+* Run updates in parallel via `GANTRY_MANIFEST_NUM_WORKERS` and `GANTRY_UPDATE_NUM_WORKERS`.
+
 | *Gantry* Env  | Purpose |
 |---------------|----------------------|
 | GANTRY_MANIFEST_CMD              | To retrieve image metadata correctly and to reduce the Docker Hub rate usage. |
@@ -62,7 +68,7 @@ The label on the services to select config to enable authentication is renamed t
 | GANTRY_REGISTRY_HOST_FILE        | To pass sensitive information via [docker secret](https://docs.docker.com/engine/swarm/secrets/). |
 | GANTRY_REGISTRY_PASSWORD_FILE    | To pass sensitive information via [docker secret](https://docs.docker.com/engine/swarm/secrets/). |
 | GANTRY_REGISTRY_USER_FILE        | To pass sensitive information via [docker secret](https://docs.docker.com/engine/swarm/secrets/). |
-| GANTRY_SERVICES_EXCLUDED_FILTERS | To provide an alternative method to exclude services from being updated. |
+| GANTRY_SERVICES_EXCLUDED_FILTERS | To provide an additional method to exclude services from being updated, in companion with `GANTRY_SERVICES_EXCLUDED`. |
 | GANTRY_UPDATE_JOBS               | *Gantry* can distinguish `replicated-job` and `global-job` from other services. *Gantry* automatically adds more options to [update services with no running tasks](faq.md#how-to-update-services-with-no-running-tasks) to avoid hanging. |
 | GANTRY_UPDATE_NUM_WORKERS        | To run multiple update commands in parallel to accelerate the updating process. |
 

From c9ef9e991e6697030320e9b65be912ad660881b0 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 19 Nov 2024 16:26:24 -0800
Subject: [PATCH 05/27] [gantry][tests] Add --with-registry-auth for more
 cases. Add gantry_login_docker_config_spec.sh

Also create and remove services in detach mode to reduce testing time.
---
 .github/workflows/on-pull-request.yml    |   1 +
 .github/workflows/on-push.yml            |   1 +
 src/lib-gantry.sh                        |  26 +-
 tests/gantry_login_docker_config_spec.sh | 306 +++++++++++++++++++++++
 tests/gantry_login_negative_spec.sh      | 133 ++++++++--
 tests/gantry_login_spec.sh               |  88 ++++++-
 tests/gantry_service_single_spec.sh      |   3 +
 tests/spec_gantry_test_helper.sh         |  26 +-
 8 files changed, 547 insertions(+), 37 deletions(-)
 create mode 100644 tests/gantry_login_docker_config_spec.sh

diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml
index 9977f1f..e9d0e88 100644
--- a/.github/workflows/on-pull-request.yml
+++ b/.github/workflows/on-pull-request.yml
@@ -37,6 +37,7 @@ jobs:
           - gantry_cleanup_images_spec.sh
           - gantry_common_options_spec.sh
           - gantry_filters_spec.sh
+          - gantry_login_docker_config_spec.sh
           - gantry_login_negative_spec.sh
           - gantry_login_spec.sh
           - gantry_manifest_spec.sh
diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml
index e4c8315..73a2097 100644
--- a/.github/workflows/on-push.yml
+++ b/.github/workflows/on-push.yml
@@ -40,6 +40,7 @@ jobs:
           - gantry_cleanup_images_spec.sh
           - gantry_common_options_spec.sh
           - gantry_filters_spec.sh
+          - gantry_login_docker_config_spec.sh
           - gantry_login_negative_spec.sh
           - gantry_login_spec.sh
           - gantry_manifest_spec.sh
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index 341bacb..382db2e 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -78,10 +78,17 @@ _read_env_or_label() {
   echo "${VALUE}"
 }
 
+_get_docker_default_config() {
+  local LOCAL_DOCKER_CONFIG="${DOCKER_CONFIG:-""}"
+  local DEFAULT_LOCATION="${LOCAL_DOCKER_CONFIG}"
+  [ -z "${DEFAULT_LOCATION}" ] && DEFAULT_LOCATION="${HOME}/.docker"
+  readlink -f "${DEFAULT_LOCATION}"
+}
+
 # Record that the default config is used.
 _use_docker_default_config() {
   local CONFIG_TO_REPORT=
-  CONFIG_TO_REPORT=$(readlink -f ~/.docker)
+  CONFIG_TO_REPORT=$(_get_docker_default_config)
   _static_variable_add_unique_to_list STATIC_VAR_DOCKER_CONFIG_DEFAULT "${CONFIG_TO_REPORT}"
 }
 
@@ -105,8 +112,8 @@ _login_registry() {
   [ -z "${PASSWORD}" ] && log ERROR "PASSWORD is empty." && return 1
   local REGISTRY_MESSAGE="registry ${HOST}"
   if [ -z "${HOST}" ]; then
-   log WARN "HOST is empty. Will login to the default registry."
-   REGISTRY_MESSAGE="default registry"
+    log WARN "HOST is empty. Will login to the default registry."
+    REGISTRY_MESSAGE="default registry"
   fi
   local AUTH_CONFIG=
   local CONFIG_MESSAGE="with default configuration"
@@ -125,7 +132,10 @@ _login_registry() {
   log INFO "Logged into ${REGISTRY_CONFIG_MESSAGE}. ${LOGIN_MSG}"
   if [ -n "${CONFIG}" ]; then
     _static_variable_add_unique_to_list STATIC_VAR_DOCKER_CONFIGS "${CONFIG}"
-  else
+  fi
+  local DEFAULT_LOCATION=
+  DEFAULT_LOCATION=$(_get_docker_default_config)
+  if [ -z "${CONFIG}" ] || [ "$(readlink -f "${CONFIG}")" = "${DEFAULT_LOCATION}" ]; then
     _use_docker_default_config
   fi
   return 0
@@ -680,10 +690,10 @@ _check_auth_config_folder() {
     return 0
   fi
   log WARN "${AUTH_CONFIG} is not a directory that contains Docker configuration files."
-  local MSG="configuration(s) set via GANTRY_REGISTRY_CONFIG* or GANTRY_REGISTRY_CONFIGS_FILE"
+  local MSG="configuration(s) set via GANTRY_REGISTRY_CONFIG or GANTRY_REGISTRY_CONFIGS_FILE"
   _report_from_static_variable STATIC_VAR_DOCKER_CONFIGS "There are" "${MSG}" "There are no ${MSG}." | log_lines WARN
   if _docker_default_config_is_used; then
-    log WARN "User logged into the default Docker configuration."
+    log WARN "User logged in using the default Docker configuration $(_get_docker_default_config)."
   fi
   return 1
 }
@@ -899,7 +909,9 @@ _get_service_update_additional_options() {
   # Add `--with-registry-auth` if needed.
   local WITH_REGISTRY_AUTH=
   WITH_REGISTRY_AUTH="$(_get_with_registry_auth "${AUTH_CONFIG}")"
-  [ -n "${WITH_REGISTRY_AUTH}" ] && OPTIONS="${OPTIONS} ${WITH_REGISTRY_AUTH}"
+  local SPACE=" "
+  [ -z "${OPTIONS}" ] && SPACE=""
+  [ -n "${WITH_REGISTRY_AUTH}" ] && OPTIONS="${OPTIONS}${SPACE}${WITH_REGISTRY_AUTH}"
   echo "${OPTIONS}"
 }
 
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
new file mode 100644
index 0000000..d6526db
--- /dev/null
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -0,0 +1,306 @@
+#!/bin/bash spellspec
+# Copyright (C) 2024 Shizun Ge
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+# Test coverage for the combination of the following 3 configurations.
+# Legend:
+# D: Whether DOCKER_CONFIG is set or not.
+# C: Whether Configuration is named or not. (i.e. GANTRY_REGISTRY_CONFIG or GANTRY_REGISTRY_CONFIGS_FILE is set or not.)
+# L: Whether GANTRY_AUTH_CONFIG_LABEL is set on the service or not.
+# (N): This is a negative test.
+# The coverage is:
+# D=0 C=0 L=0 test_login_default_config
+# D=0 C=0 L=1 test_login_config_mismatch (N)
+# D=0 C=1 L=0 test_login_multi_services_no_label SERVICE_NAME0 (N)
+# D=0 C=1 L=1 test_login_config
+# D=1 C=0 L=0 test_login_docker_config_default_config
+# D=1 C=0 L=1 test_login_docker_config_label_override SERVICE_NAME0 (N)
+# D=1 C=1 L=0 test_login_docker_config_no_label
+# D=1 C=1 L=1 test_login_docker_config_label_override SERVICE_NAME1
+
+Describe 'login_docker_config'
+  SUITE_NAME="login_docker_config"
+  BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
+  AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
+  Describe "test_login_docker_config_no_label" "container_test:true"
+    TEST_NAME="test_login_docker_config_no_label"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    CONFIG="C$(unique_id)"
+    TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
+    test_login_docker_config_no_label() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      local CONFIG="${3}"
+      local REGISTRY="${4}"
+      local USERNAME="${5}"
+      local PASSWORD="${6}"
+      check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
+      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      reset_gantry_env "${SERVICE_NAME}"
+      export DOCKER_CONFIG="${CONFIG}"
+      export GANTRY_REGISTRY_CONFIG="${CONFIG}"
+      export GANTRY_REGISTRY_HOST="${REGISTRY}"
+      export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
+      export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
+      local RETURN_VALUE=
+      run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      rm "${USER_FILE}"
+      rm "${PASS_FILE}"
+      [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
+      return "${RETURN_VALUE}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
+      The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      # No --config, due to GANTRY_AUTH_CONFIG_LABEL is not found on the service.
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
+      # Gantry adds --with-registry-auth, because DOCKER_CONFIG and the configuration name are same (default location).
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_login_docker_config_default_config" "container_test:true"
+    TEST_NAME="test_login_docker_config_default_config"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    CONFIG="C$(unique_id)"
+    TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
+    test_login_docker_config_default_config() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      local CONFIG="${3}"
+      local REGISTRY="${4}"
+      local USERNAME="${5}"
+      local PASSWORD="${6}"
+      check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
+      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      # Do not set GANTRY_AUTH_CONFIG_LABEL on the service.
+      reset_gantry_env "${SERVICE_NAME}"
+      export DOCKER_CONFIG="${CONFIG}"
+      # Do not set GANTRY_REGISTRY_CONFIG to login to the default configuration.
+      export GANTRY_REGISTRY_HOST="${REGISTRY}"
+      export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
+      export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
+      local RETURN_VALUE=
+      run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      docker logout "${REGISTRY}" > /dev/null
+      rm "${USER_FILE}"
+      rm "${PASS_FILE}"
+      [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
+      return "${RETURN_VALUE}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
+      The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
+      # Gantry adds --with-registry-auth for using the default configuration.
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_login_docker_config_label_override" "container_test:false"
+    TEST_NAME="test_login_docker_config_label_override"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME0="${SERVICE_NAME}-0"
+    SERVICE_NAME1="${SERVICE_NAME}-1"
+    SERVICE_NAME2="${SERVICE_NAME}-2"
+    CONFIG="C$(unique_id)"
+    TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
+    test_start() {
+      local TEST_NAME="${1}"
+      local IMAGE_WITH_TAG="${2}"
+      local SERVICE_NAME="${3}"
+      local SERVICE_NAME0="${SERVICE_NAME}-0"
+      local SERVICE_NAME1="${SERVICE_NAME}-1"
+      local SERVICE_NAME2="${SERVICE_NAME}-2"
+      initialize_test "${TEST_NAME}"
+      build_and_push_test_image "${IMAGE_WITH_TAG}"
+      start_replicated_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG}"
+      start_replicated_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG}"
+      start_replicated_service "${SERVICE_NAME2}" "${IMAGE_WITH_TAG}"
+      build_and_push_test_image "${IMAGE_WITH_TAG}"
+    }
+    test_login_docker_config_label_override() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      local CONFIG="${3}"
+      local REGISTRY="${4}"
+      local USERNAME="${5}"
+      local PASSWORD="${6}"
+      local SERVICE_NAME0="${SERVICE_NAME}-0"
+      local SERVICE_NAME1="${SERVICE_NAME}-1"
+      local SERVICE_NAME2="${SERVICE_NAME}-2"
+      local INCORRECT_CONFIG="incorrect-${CONFIG}"
+      check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
+      local CONFIGS_FILE=
+      CONFIGS_FILE=$(mktemp)
+      echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" > "${CONFIGS_FILE}"
+      # Set GANTRY_AUTH_CONFIG_LABEL on SERVICE_NAME1, but not on SERVICE_NAME0.
+      # Inspection of SERVICE_NAME0 should fail (incorrect label overrides DOCKER_CONFIG).
+      # Inspection of SERVICE_NAME1 should pass (correct label overrides DOCKER_CONFIG).
+      # Inspection of SERVICE_NAME2 should pass (use DOCKER_CONFIG).
+      docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME0}"
+      docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME1}"
+      reset_gantry_env "${SERVICE_NAME}"
+      # Inspection and updating should use DOCKER_CONFIG or the label.
+      export DOCKER_CONFIG="${CONFIG}"
+      export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
+      # Set GANTRY_CLEANUP_IMAGES="false" to speedup the test. We are not testing removing image here.
+      export GANTRY_CLEANUP_IMAGES="false"
+      local RETURN_VALUE=
+      run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      rm "${CONFIGS_FILE}"
+      [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
+      [ -d "${INCORRECT_CONFIG}" ] && rm -r "${INCORRECT_CONFIG}"
+      return "${RETURN_VALUE}"
+    }
+    test_end() {
+      local TEST_NAME="${1}"
+      local IMAGE_WITH_TAG="${2}"
+      local SERVICE_NAME="${3}"
+      local SERVICE_NAME0="${SERVICE_NAME}-0"
+      local SERVICE_NAME1="${SERVICE_NAME}-1"
+      local SERVICE_NAME2="${SERVICE_NAME}-2"
+      stop_service "${SERVICE_NAME0}"
+      stop_service "${SERVICE_NAME1}"
+      stop_service "${SERVICE_NAME2}"
+      prune_local_test_image "${IMAGE_WITH_TAG}"
+      finalize_test "${TEST_NAME}"
+    }
+    BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      The status should be failure
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      # Check warnings
+      The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
+      The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME2}"
+      The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*"
+      The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME1}.*"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME2}.*${PERFORM_REASON_KNOWN_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME2}.*"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      # No --with-registry-auth, due to the incorrect configuration, image inspection failed, we did not reach the updating step.
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME0}"
+      # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME1}"
+      # Gantry adds --with-registry-auth, because DOCKER_CONFIG and the configuration name are same (default location).
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME2}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME0}.*"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME1}.*"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME2}.*"
+      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME2}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME2}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "2 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
+    End
+  End
+End # Describe 'login_docker_config'
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index 6ec21d6..61b8189 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -50,7 +50,9 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
+      # No --with-registry-auth, because no login.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -118,6 +120,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -187,6 +190,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -203,13 +207,13 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_config_mismatch" "container_test:false"
-    TEST_NAME="test_login_config_mismatch"
+  Describe "test_login_config_mismatch_default" "container_test:false"
+    TEST_NAME="test_login_config_mismatch_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
-    test_login_config_mismatch() {
+    test_login_config_mismatch_default() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local CONFIG="${3}"
@@ -217,7 +221,7 @@ Describe 'login_negative'
       local USERNAME="${5}"
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local INCORRECT_CONFIG="${CONFIG}-incorrect"
+      local INCORRECT_CONFIG="incorrect-${CONFIG}"
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       # Also use CONFIGS_FILE to test a explicitly-set config.
@@ -247,7 +251,7 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_config_mismatch "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -256,19 +260,99 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
-      The stderr should satisfy spec_expect_message    "There are 1 configuration\(s\).*"
-      The stderr should satisfy spec_expect_message    "User logged into the default Docker config.*"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*\"--config ${CONFIG}\".*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*\"--config ${CONFIG}-incorrect\".*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
+      The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
+      # No --with-registry-auth, due to the incorrect configuration, image inspection failed, we did not reach the updating step.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_login_config_mismatch_no_default" "container_test:false"
+    TEST_NAME="test_login_config_mismatch_no_default"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    CONFIG="C$(unique_id)"
+    TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
+    test_login_config_mismatch_no_default() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      local CONFIG="${3}"
+      local REGISTRY="${4}"
+      local USERNAME="${5}"
+      local PASSWORD="${6}"
+      local INCORRECT_CONFIG="incorrect-${CONFIG}"
+      check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
+      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      # The config name on the service is different from the config name used in GANTRY_REGISTRY_CONFIG
+      docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME}"
+      reset_gantry_env "${SERVICE_NAME}"
+      export GANTRY_REGISTRY_CONFIG="${CONFIG}"
+      export GANTRY_REGISTRY_HOST="${REGISTRY}"
+      export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
+      export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
+      export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
+      local RETURN_VALUE=
+      run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      rm "${USER_FILE}"
+      rm "${PASS_FILE}"
+      [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
+      [ -d "${INCORRECT_CONFIG}" ] && rm -r "${INCORRECT_CONFIG}"
+      return "${RETURN_VALUE}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      The status should be failure
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      # Check warnings
+      The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
+      # This message does not present, because we don't login with the default configuration.
+      The stderr should satisfy spec_expect_no_message "${USER_LOGGED_INTO_DEFAULT}.*"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
+      # No --with-registry-auth, due to the incorrect configuration, image inspection failed, we did not reach the updating step.
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -319,19 +403,22 @@ Describe 'login_negative'
       local REGISTRY="${4}"
       local USERNAME="${5}"
       local PASSWORD="${6}"
+      local SERVICE_NAME0="${SERVICE_NAME}-0"
+      local SERVICE_NAME1="${SERVICE_NAME}-1"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
       # Set GANTRY_AUTH_CONFIG_LABEL on SERVICE_NAME1, but not on SERVICE_NAME0.
-      # Inspection of SERVICE_NAME0 should fail.
+      # Inspection of SERVICE_NAME0 should fail, because GANTRY_AUTH_CONFIG_LABEL is not found.
+      # Inspection of SERVICE_NAME1 should pass, because configuration is set via GANTRY_AUTH_CONFIG_LABEL.
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME1}"
       reset_gantry_env "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
+      # Set GANTRY_CLEANUP_IMAGES="false" to speedup the test. We are not testing removing image here.
+      export GANTRY_CLEANUP_IMAGES="false"
       local RETURN_VALUE=
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
@@ -348,10 +435,10 @@ Describe 'login_negative'
       local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
       local SERVICE_NAME0="${SERVICE_NAME}-0"
       local SERVICE_NAME1="${SERVICE_NAME}-1"
-      stop_service "${SERVICE_NAME1}"
       stop_service "${SERVICE_NAME0}"
-      prune_local_test_image "${IMAGE_WITH_TAG1}"
+      stop_service "${SERVICE_NAME1}"
       prune_local_test_image "${IMAGE_WITH_TAG0}"
+      prune_local_test_image "${IMAGE_WITH_TAG1}"
       finalize_test "${TEST_NAME}"
     }
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -375,8 +462,12 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      # No --with-registry-auth, for 1. no label on the SERVICE_NAME0. 2. GANTRY_REGISTRY_CONFIG is set but it is not same as the default location.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME0}"
+      # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME0}.*"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME1}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME0}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME1}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME0}"
@@ -389,12 +480,10 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATE_FAILED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
       The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG0}"
-      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG1}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG0}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG1}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
     End
   End
   Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false"
@@ -449,6 +538,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -513,6 +603,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index e6bc5ff..3e9e0c5 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -15,6 +15,9 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+# This warning is generated by a docker command.
+export IMAGE_DIGEST_WARNING="image .* could not be accessed on a registry to record its digest. Each node will access .* independently, possibly leading to different nodes running different versions of the image"
+
 Describe 'login'
   SUITE_NAME="login"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
@@ -41,6 +44,9 @@ Describe 'login'
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
+      # To test duplicated "--with-registry-auth".
+      # A duplicated "--with-registry-auth" will be added, but it should be ok.
+      export GANTRY_UPDATE_OPTIONS="--with-registry-auth"
       local RETURN_VALUE=
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
@@ -69,7 +75,9 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      # 2 "--with-registry-auth" for SERVICE_NAME. One is automatically added. The other is from user.
+      The stderr should satisfy spec_expect_multiple_messages "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -135,9 +143,10 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}"
-      # When using the default configuration, Gantry should still add --with-registry-auth.
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
+      # Gantry adds --with-registry-auth for using the default configuration.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -209,7 +218,80 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}"
+      # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_login_external_config" "container_test:true"
+    TEST_NAME="test_login_external_config"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    CONFIG="$(mktemp -d)"
+    TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
+    test_login_external_config() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      local CONFIG="${3}"
+      local REGISTRY="${4}"
+      local USERNAME="${5}"
+      local PASSWORD="${6}"
+      check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
+      # Login outside Gantry.
+      echo "${PASSWORD}" | docker --config "${CONFIG}" login --username="${USERNAME}" --password-stdin "${REGISTRY}" > /dev/null 2>&1
+      chmod 555 "${CONFIG}"
+      # Do not set GANTRY_AUTH_CONFIG_LABEL on service.
+      reset_gantry_env "${SERVICE_NAME}"
+      export DOCKER_CONFIG="${CONFIG}"
+      # Use manifest to avoid write to DOCKER_CONFIG.
+      # When running container test, Gantry runs as root.
+      # We cannot remove DOCKER_CONFIG when it containes data from root.
+      export GANTRY_MANIFEST_CMD="manifest"
+      export GANTRY_MANIFEST_OPTIONS="--insecure"
+      # Do not set --with-registry-auth to trigger a warning IMAGE_DIGEST_WARNING.
+      local RETURN_VALUE=
+      run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      [ -d "${CONFIG}" ] && chmod 777 "${CONFIG}" && rm -r "${CONFIG}"
+      return "${RETURN_VALUE}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_login_external_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
+      The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
+      # No --with-registry-auth.
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*"
+      # Check the warning due to missing --with-registry-auth.
+      The stderr should satisfy spec_expect_message    "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*${IMAGE_DIGEST_WARNING}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index c9277fd..edc5c20 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -46,6 +46,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -90,6 +91,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -144,6 +146,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 237a2de..195aaa8 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -41,10 +41,13 @@ export PERFORM_REASON_DIGEST_IS_EMPTY="because DIGEST is empty"
 export PERFORM_REASON_HAS_NEWER_IMAGE="because there is a newer version"
 export IMAGE_NOT_EXIST="does not exist or it is not available"
 export CONFIG_IS_NOT_A_DIRECTORY="is not a directory that contains Docker configuration files"
+export THERE_ARE_NUM_CONFIGURATIONS="There are [0-9]+ configuration\(s\)"
+export USER_LOGGED_INTO_DEFAULT="User logged in using the default Docker configuration"
 export ADDING_OPTIONS="Adding options"
 export ADDING_OPTIONS_WITH_REGISTRY_AUTH="Adding options.*--with-registry-auth"
 export SET_TIMEOUT_TO="Set timeout to"
 export RETURN_VALUE_INDICATES_TIMEOUT="The return value 124 indicates the job timed out."
+export THERE_ARE_ADDITIONAL_MESSAGES="There are additional messages from updating"
 export NUM_SERVICES_SKIP_JOBS="Skip updating [0-9]+ service\(s\) due to they are job\(s\)"
 export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)"
 export NUM_SERVICES_NO_NEW_IMAGES="No new images for [0-9]+ service\(s\)"
@@ -403,6 +406,7 @@ initialize_test() {
 
 reset_gantry_env() {
   local SERVICE_NAME="${1}"
+  export DOCKER_CONFIG=
   export DOCKER_HOST=
   export GANTRY_LOG_LEVEL="DEBUG"
   export GANTRY_NODE_NAME=
@@ -572,7 +576,7 @@ build_and_push_test_image() {
 prune_local_test_image() {
   local IMAGE_WITH_TAG="${1}"
   echo -n "Removing image ${IMAGE_WITH_TAG} "
-  docker image rm "${IMAGE_WITH_TAG}" --force
+  docker image rm "${IMAGE_WITH_TAG}" --force &
 }
 
 docker_service_update() {
@@ -681,6 +685,7 @@ start_replicated_service() {
   local IMAGE_WITH_TAG="${2}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "Creating service ${SERVICE_NAME} in replicated mode "
+  # Add --detach to reduce the test runtime.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
   timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
@@ -690,6 +695,7 @@ start_replicated_service() {
     --with-registry-auth \
     $(_location_constraints) \
     --mode=replicated \
+    --detach=true \
     "${IMAGE_WITH_TAG}"
 }
 
@@ -698,6 +704,7 @@ start_global_service() {
   local IMAGE_WITH_TAG="${2}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "Creating service ${SERVICE_NAME} in global mode "
+  # Do not add --detach, because we want to wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
   timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
@@ -715,6 +722,7 @@ _start_replicated_job() {
   local IMAGE_WITH_TAG="${2}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "Creating service ${SERVICE_NAME} in replicated job mode "
+  # Always add --detach=true, do not wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
   timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
@@ -723,7 +731,8 @@ _start_replicated_job() {
     --restart-max-attempts 5 \
     --with-registry-auth \
     $(_location_constraints) \
-    --mode=replicated-job --detach=true \
+    --mode=replicated-job \
+    --detach=true \
     "${IMAGE_WITH_TAG}"
   # wait until the job is running
   _wait_service_state "${SERVICE_NAME}" "Running"
@@ -732,7 +741,7 @@ _start_replicated_job() {
 stop_service() {
   local SERVICE_NAME="${1}"
   echo -n "Removing service "
-  docker service rm "${SERVICE_NAME}"
+  docker service rm "${SERVICE_NAME}" &
 }
 
 _get_script_dir() {
@@ -769,9 +778,12 @@ _get_entrypoint() {
 
 _add_file_to_mount_options() {
   local MOUNT_OPTIONS="${1}"
-  local FILE="${2}"
-  if [ -n "${FILE}" ] && [ -r "${FILE}" ]; then
-    MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${FILE},target=${FILE}"
+  local HOST_PATH="${2}"
+  if [ -n "${HOST_PATH}" ] && [ -r "${HOST_PATH}" ]; then
+    # Use the absolute path inside the container.
+    local TARGET=
+    TARGET=$(readlink -f "${HOST_PATH}")
+    MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${HOST_PATH},target=${TARGET}"
   fi
   echo "${MOUNT_OPTIONS}"
 }
@@ -787,6 +799,7 @@ _run_gantry_container() {
   SERVICE_NAME="gantry-test-SUT-$(unique_id)"
   docker service rm "${SERVICE_NAME}" >/dev/null 2>&1;
   local MOUNT_OPTIONS=
+  MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${DOCKER_CONFIG}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_CONFIG_FILE}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_CONFIGS_FILE}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_HOST_FILE}")
@@ -802,6 +815,7 @@ _run_gantry_container() {
     --constraint "node.role==manager" \
     --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
     ${MOUNT_OPTIONS} \
+    --env "DOCKER_CONFIG=${DOCKER_CONFIG}" \
     --env "DOCKER_HOST=${DOCKER_HOST}" \
     --env "GANTRY_LOG_LEVEL=${GANTRY_LOG_LEVEL}" \
     --env "GANTRY_NODE_NAME=${GANTRY_NODE_NAME}" \

From 55ec2918412a6e9ea79d2abce300e99f3e7c9a4b Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 19 Nov 2024 21:03:20 -0800
Subject: [PATCH 06/27] [tests] add start_multiple_replicated_services and
 stop_multiple_services

---
 tests/gantry_cleanup_images_spec.sh      |  5 +--
 tests/gantry_login_docker_config_spec.sh | 14 +-----
 tests/gantry_login_negative_spec.sh      |  5 +--
 tests/gantry_parallel_spec.sh            | 11 +----
 tests/gantry_service_multiple_spec.sh    | 27 ++----------
 tests/spec_gantry_test_helper.sh         | 55 +++++++++++++++---------
 6 files changed, 43 insertions(+), 74 deletions(-)

diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index b05629d..b2589aa 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -200,10 +200,7 @@ Describe 'cleanup-images'
       local SERVICE_NAME="${3}"
       local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0"
       local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
-      stop_service "${SERVICE_NAME0}"
-      stop_service "${SERVICE_NAME1}"
+      stop_multiple_services "${SERVICE_NAME}" 0 1
       # If run successfully, IMAGE_WITH_TAG0 should already be removed.
       prune_local_test_image "${IMAGE_WITH_TAG0}" 2>&1
       prune_local_test_image "${IMAGE_WITH_TAG1}"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index d6526db..e487cc3 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -188,14 +188,9 @@ Describe 'login_docker_config'
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
       local SERVICE_NAME="${3}"
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
-      local SERVICE_NAME2="${SERVICE_NAME}-2"
       initialize_test "${TEST_NAME}"
       build_and_push_test_image "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME2}" "${IMAGE_WITH_TAG}"
+      start_multiple_replicated_services "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 0 2
       build_and_push_test_image "${IMAGE_WITH_TAG}"
     }
     test_login_docker_config_label_override() {
@@ -237,12 +232,7 @@ Describe 'login_docker_config'
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
       local SERVICE_NAME="${3}"
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
-      local SERVICE_NAME2="${SERVICE_NAME}-2"
-      stop_service "${SERVICE_NAME0}"
-      stop_service "${SERVICE_NAME1}"
-      stop_service "${SERVICE_NAME2}"
+      stop_multiple_services "${SERVICE_NAME}" 0 2
       prune_local_test_image "${IMAGE_WITH_TAG}"
       finalize_test "${TEST_NAME}"
     }
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index 61b8189..ef8b59b 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -433,10 +433,7 @@ Describe 'login_negative'
       local SERVICE_NAME="${3}"
       local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0"
       local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
-      stop_service "${SERVICE_NAME0}"
-      stop_service "${SERVICE_NAME1}"
+      stop_multiple_services "${SERVICE_NAME}" 0 1
       prune_local_test_image "${IMAGE_WITH_TAG0}"
       prune_local_test_image "${IMAGE_WITH_TAG1}"
       finalize_test "${TEST_NAME}"
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index 6271a7e..96770a0 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -35,16 +35,7 @@ Describe 'service-parallel'
       common_setup_new_image_multiple "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME}" "${MAX_SERVICES_NUM}"
       local NO_NEW_IAMGE_START=$((MAX_SERVICES_NUM+1))
       local NO_NEW_IAMGE_END=$((MAX_SERVICES_NUM+MAX_NO_NEW_IMAGE))
-      local NUM=
-      local PIDS=
-      for NUM in $(seq "${NO_NEW_IAMGE_START}" "${NO_NEW_IAMGE_END}"); do
-        local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
-        start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" &
-        PIDS="${!} ${PIDS}"
-      done
-      # SC2086 (info): Double quote to prevent globbing and word splitting.
-      # shellcheck disable=SC2086
-      wait ${PIDS}
+      start_multiple_replicated_services "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${NO_NEW_IAMGE_START}" "${NO_NEW_IAMGE_END}"
     }
     test_parallel_less_workers() {
       local TEST_NAME="${1}"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index 90ac14e..9e0b17f 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -33,22 +33,12 @@ Describe 'service-multiple-services'
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
       local SERVICE_NAME="${3}"
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
-      local SERVICE_NAME2="${SERVICE_NAME}-2"
-      local SERVICE_NAME3="${SERVICE_NAME}-3"
-      local SERVICE_NAME4="${SERVICE_NAME}-4"
-      local SERVICE_NAME5="${SERVICE_NAME}-5"
 
       initialize_test "${TEST_NAME}"
       build_and_push_test_image "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME2}" "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME3}" "${IMAGE_WITH_TAG}"
+      start_multiple_replicated_services "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 0 3
       build_and_push_test_image "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME4}" "${IMAGE_WITH_TAG}"
-      start_replicated_service "${SERVICE_NAME5}" "${IMAGE_WITH_TAG}"
+      start_multiple_replicated_services "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 4 5
     }
     test_multiple_services_excluded_filters() {
       local TEST_NAME="${1}"
@@ -65,18 +55,7 @@ Describe 'service-multiple-services'
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
       local SERVICE_NAME="${3}"
-      local SERVICE_NAME0="${SERVICE_NAME}-0"
-      local SERVICE_NAME1="${SERVICE_NAME}-1"
-      local SERVICE_NAME2="${SERVICE_NAME}-2"
-      local SERVICE_NAME3="${SERVICE_NAME}-3"
-      local SERVICE_NAME4="${SERVICE_NAME}-4"
-      local SERVICE_NAME5="${SERVICE_NAME}-5"
-      stop_service "${SERVICE_NAME5}"
-      stop_service "${SERVICE_NAME4}"
-      stop_service "${SERVICE_NAME3}"
-      stop_service "${SERVICE_NAME2}"
-      stop_service "${SERVICE_NAME1}"
-      stop_service "${SERVICE_NAME0}"
+      stop_multiple_services "${SERVICE_NAME}" 0 5
       prune_local_test_image "${IMAGE_WITH_TAG}"
       finalize_test "${TEST_NAME}"
     }
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 195aaa8..ffdbb53 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -108,16 +108,7 @@ common_setup_new_image_multiple() {
   local MAX_SERVICES_NUM="${4}"
   initialize_test "${TEST_NAME}"
   build_and_push_test_image "${IMAGE_WITH_TAG}"
-  local NUM=
-  local PIDS=
-  for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do
-    local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
-    start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" &
-    PIDS="${!} ${PIDS}"
-  done
-  # SC2086 (info): Double quote to prevent globbing and word splitting.
-  # shellcheck disable=SC2086
-  wait ${PIDS}
+  start_multiple_replicated_services "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 0 "${MAX_SERVICES_NUM}"
   build_and_push_test_image "${IMAGE_WITH_TAG}"
 }
 
@@ -171,16 +162,7 @@ common_cleanup_multiple() {
   local IMAGE_WITH_TAG="${2}"
   local SERVICE_NAME="${3}"
   local MAX_SERVICES_NUM="${4}"
-  local NUM=
-  local PIDS=
-  for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do
-    local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
-    stop_service "${SERVICE_NAME_NUM}" &
-    PIDS="${!} ${PIDS}"
-  done
-  # SC2086 (info): Double quote to prevent globbing and word splitting.
-  # shellcheck disable=SC2086
-  wait ${PIDS}
+  stop_multiple_services "${SERVICE_NAME}" 0 "${MAX_SERVICES_NUM}"
   prune_local_test_image "${IMAGE_WITH_TAG}"
   finalize_test "${TEST_NAME}"
 }
@@ -699,6 +681,23 @@ start_replicated_service() {
     "${IMAGE_WITH_TAG}"
 }
 
+start_multiple_replicated_services() {
+  local SERVICE_NAME="${1}"
+  local IMAGE_WITH_TAG="${2}"
+  local START_NUM="${3}"
+  local END_NUM="${4}"
+  local NUM=
+  local PIDS=
+  for NUM in $(seq "${START_NUM}" "${END_NUM}"); do
+    local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
+    start_replicated_service "${SERVICE_NAME_NUM}" "${IMAGE_WITH_TAG}" &
+    PIDS="${!} ${PIDS}"
+  done
+  # SC2086 (info): Double quote to prevent globbing and word splitting.
+  # shellcheck disable=SC2086
+  wait ${PIDS}
+}
+
 start_global_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
@@ -744,6 +743,22 @@ stop_service() {
   docker service rm "${SERVICE_NAME}" &
 }
 
+stop_multiple_services() {
+  local SERVICE_NAME="${1}"
+  local START_NUM="${2}"
+  local END_NUM="${3}"
+  local NUM=
+  local PIDS=
+  for NUM in $(seq "${START_NUM}" "${END_NUM}"); do
+    local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
+    stop_service "${SERVICE_NAME_NUM}" &
+    PIDS="${!} ${PIDS}"
+  done
+  # SC2086 (info): Double quote to prevent globbing and word splitting.
+  # shellcheck disable=SC2086
+  wait ${PIDS}
+}
+
 _get_script_dir() {
   # SC2128: Expanding an array without an index only gives the first element.
   # SC3054 (warning): In POSIX sh, array references are undefined.

From 466857d92fe88a1dec6bb6154f7027c46b90381f Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 19 Nov 2024 21:29:31 -0800
Subject: [PATCH 07/27] [tests] add coverage tag

---
 .github/workflows/coverage.yml                |  2 +-
 tests/README.md                               |  2 +-
 tests/gantry_cleanup_images_spec.sh           |  8 ++++----
 tests/gantry_common_options_spec.sh           | 13 +++++++------
 tests/gantry_filters_spec.sh                  |  8 ++++----
 tests/gantry_login_docker_config_spec.sh      |  6 +++---
 tests/gantry_login_negative_spec.sh           | 16 ++++++++--------
 tests/gantry_login_spec.sh                    |  8 ++++----
 tests/gantry_manifest_spec.sh                 | 12 ++++++------
 tests/gantry_notify_spec.sh                   | 12 ++++++------
 tests/gantry_parallel_spec.sh                 |  8 ++++----
 tests/gantry_rollback_spec.sh                 | 12 ++++++------
 tests/gantry_service_multiple_spec.sh         |  2 +-
 tests/gantry_service_no_running_tasks_spec.sh |  4 ++--
 tests/gantry_service_single_spec.sh           |  6 +++---
 tests/gantry_update_options_spec.sh           | 14 +++++++-------
 16 files changed, 67 insertions(+), 66 deletions(-)

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index da8b136..9d5e6c3 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -38,7 +38,7 @@ jobs:
         run: |
           export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
           export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
-          bash shellspec --kcov
+          bash shellspec --kcov --jobs 50 --tag "coverage:true"
       - name: Upload reports
         uses: actions/upload-artifact@v4
         with:
diff --git a/tests/README.md b/tests/README.md
index 434a5f0..1da24c1 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -35,7 +35,7 @@ bash shellspec --kcov
 If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
 ```
 export GANTRY_TEST_CONTAINER_REPO_TAG=<gantry image>:<tag>
-bash shellspec --tag "container_test:true"
+bash shellspec --tag "container_test:true" "coverage:true"
 ```
 
 > NOTE: Negative tests will hang when testing a *Gantry* container, which may be due to a bug in shellspec. So when testing *Gantry* images, we should run only tests with tag `container_test:true`.
diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index b2589aa..9613f4b 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -20,7 +20,7 @@ Describe 'cleanup-images'
   SUITE_NAME="cleanup-images"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_CLEANUP_IMAGES_false" "container_test:true"
+  Describe "test_CLEANUP_IMAGES_false" "container_test:true" "coverage:true"
     TEST_NAME="test_CLEANUP_IMAGES_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -62,7 +62,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true"
+  Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true" "coverage:true"
     TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -108,7 +108,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true"
+  Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true" "coverage:true"
     TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_good"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -153,7 +153,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true"
+  Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" "coverage:true"
     # Test the remove image entrypoint. To improve coverage.
     TEST_NAME="test_IMAGES_TO_REMOVE_none_empty"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index f64bd55..bb3e7f2 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -19,7 +19,7 @@ Describe 'common-options'
   SUITE_NAME="common-options"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false"
+  Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true"
     TEST_NAME="test_common_DOCKER_HOST_not_swarm_manager"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -68,7 +68,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
   End
-  Describe "test_common_LOG_LEVEL_none" "container_test:true"
+  Describe "test_common_LOG_LEVEL_none" "container_test:true" "coverage:true"
     TEST_NAME="test_common_LOG_LEVEL_none"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -91,7 +91,8 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message ".+"
     End
   End
-  Describe "test_common_no_new_env" "container_test:false"
+  # Do not run test_common_no_new_env with the kcov, which alters the environment variables.
+  Describe "test_common_no_new_env" "container_test:false" "coverage:false"
     # Check there is no new variable set,
     # to avoid errors like https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
     # We don't need to run this test using containers because we check env on the host, while the container test set env inside the container.
@@ -134,7 +135,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message ".+"
     End
   End
-  Describe "test_common_PRE_POST_RUN_CMD" "container_test:true"
+  Describe "test_common_PRE_POST_RUN_CMD" "container_test:true" "coverage:true"
     TEST_NAME="test_common_PRE_POST_RUN_CMD"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -184,7 +185,7 @@ Describe 'common-options'
   # run_gantry prints logs after gantry exists, while testing a container.
   # In thes test, gantry never exit, but will be killed, thus there is no log.
   # Therefore we disable the container test for this test.
-  Describe "test_common_SLEEP_SECONDS" "container_test:false"
+  Describe "test_common_SLEEP_SECONDS" "container_test:false" "coverage:true"
     TEST_NAME="test_common_SLEEP_SECONDS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -233,7 +234,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_message           "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
   End
-  Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false"
+  Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_common_SLEEP_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index f4f7a1f..819cd4b 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -19,7 +19,7 @@ Describe 'filters'
   SUITE_NAME="filters"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_SERVICES_FILTERS_bad" "container_test:false"
+  Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
     TEST_NAME="test_SERVICES_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -63,7 +63,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true"
+  Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_multiple_services"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -120,7 +120,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_FILTERS_default" "container_test:true"
+  Describe "test_SERVICES_EXCLUDED_FILTERS_default" "container_test:true" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -170,7 +170,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false"
+  Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index e487cc3..64c0714 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -35,7 +35,7 @@ Describe 'login_docker_config'
   SUITE_NAME="login_docker_config"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
-  Describe "test_login_docker_config_no_label" "container_test:true"
+  Describe "test_login_docker_config_no_label" "container_test:true" "coverage:true"
     TEST_NAME="test_login_docker_config_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -105,7 +105,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_docker_config_default_config" "container_test:true"
+  Describe "test_login_docker_config_default_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_docker_config_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -175,7 +175,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_docker_config_label_override" "container_test:false"
+  Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
     TEST_NAME="test_login_docker_config_label_override"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index ef8b59b..809ec64 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -19,7 +19,7 @@ Describe 'login_negative'
   SUITE_NAME="login_negative"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
-  Describe "test_login_no_login" "container_test:false"
+  Describe "test_login_no_login" "container_test:false" "coverage:true"
     TEST_NAME="test_login_no_login"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -69,7 +69,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_incorrect_password" "container_test:false"
+  Describe "test_login_incorrect_password" "container_test:false" "coverage:true"
     TEST_NAME="test_login_incorrect_password"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -137,7 +137,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_read_only_file" "container_test:false"
+  Describe "test_login_read_only_file" "container_test:false" "coverage:true"
     TEST_NAME="test_login_read_only_file"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -207,7 +207,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_config_mismatch_default" "container_test:false"
+  Describe "test_login_config_mismatch_default" "container_test:false" "coverage:true"
     TEST_NAME="test_login_config_mismatch_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -291,7 +291,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_config_mismatch_no_default" "container_test:false"
+  Describe "test_login_config_mismatch_no_default" "container_test:false" "coverage:true"
     TEST_NAME="test_login_config_mismatch_no_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -369,7 +369,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_multi_services_no_label" "container_test:false"
+  Describe "test_login_multi_services_no_label" "container_test:false" "coverage:true"
     # To test https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
     TEST_NAME="test_login_multi_services_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
@@ -483,7 +483,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
     End
   End
-  Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false"
+  Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" "coverage:true"
     TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -552,7 +552,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_file_not_exist" "container_test:false"
+  Describe "test_login_file_not_exist" "container_test:false" "coverage:true"
     TEST_NAME="test_login_file_not_exist"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 3e9e0c5..11cea48 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -22,7 +22,7 @@ Describe 'login'
   SUITE_NAME="login"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
-  Describe "test_login_config" "container_test:true"
+  Describe "test_login_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -94,7 +94,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_default_config" "container_test:true"
+  Describe "test_login_default_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -163,7 +163,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_REGISTRY_CONFIGS_FILE" "container_test:true"
+  Describe "test_login_REGISTRY_CONFIGS_FILE" "container_test:true" "coverage:true"
     TEST_NAME="test_login_REGISTRY_CONFIGS_FILE"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -237,7 +237,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_login_external_config" "container_test:true"
+  Describe "test_login_external_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_external_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index 278e560..ada408a 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -19,7 +19,7 @@ Describe 'manifest-command'
   SUITE_NAME="manifest-command"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_MANIFEST_CMD_none" "container_test:true"
+  Describe "test_MANIFEST_CMD_none" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_none"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -69,7 +69,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_MANIFEST_CMD_none_SERVICES_SELF" "container_test:true"
+  Describe "test_MANIFEST_CMD_none_SERVICES_SELF" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_none_SERVICES_SELF"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -116,7 +116,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_MANIFEST_CMD_manifest" "container_test:true"
+  Describe "test_MANIFEST_CMD_manifest" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_manifest"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -160,7 +160,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_MANIFEST_CMD_label" "container_test:true"
+  Describe "test_MANIFEST_CMD_label" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -209,7 +209,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false"
+  Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_unsupported_cmd"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -255,7 +255,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_MANIFEST_CMD_failure" "container_test:false"
+  Describe "test_MANIFEST_CMD_failure" "container_test:false" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_failure"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index 6ddd76b..b3fd52f 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -70,7 +70,7 @@ Describe 'notify'
   SUITE_NAME="notify"
   BeforeAll "_notify_before_all ${SUITE_NAME}"
   AfterAll "_notify_after_all ${SUITE_NAME}"
-  Describe "test_notify_apprise" "container_test:true"
+  Describe "test_notify_apprise" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_apprise"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -123,7 +123,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
   End
-  Describe "test_notify_apprise_no_new_image" "container_test:true"
+  Describe "test_notify_apprise_no_new_image" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_apprise_no_new_image"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -177,7 +177,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
   End
-  Describe "test_notify_apprise_bad_url" "container_test:true"
+  Describe "test_notify_apprise_bad_url" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_apprise_bad_url"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -221,7 +221,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "Failed to send notification via Apprise"
     End
   End
-  Describe "test_notify_on_change_new_image" "container_test:true"
+  Describe "test_notify_on_change_new_image" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_on_change_new_image"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -275,7 +275,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
   End
-  Describe "test_notify_on_change_no_updates" "container_test:true"
+  Describe "test_notify_on_change_no_updates" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_on_change_no_updates"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -325,7 +325,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SKIP_SENDING_NOTIFICATION}"
     End
   End
-  Describe "test_notify_on_change_errors" "container_test:false"
+  Describe "test_notify_on_change_errors" "container_test:false" "coverage:true"
     TEST_NAME="test_notify_on_change_errors"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index 96770a0..34b7e4c 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -19,7 +19,7 @@ Describe 'service-parallel'
   SUITE_NAME="service-parallel"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_parallel_less_workers" "container_test:true"
+  Describe "test_parallel_less_workers" "container_test:true" "coverage:true"
     TEST_NAME="test_parallel_less_workers"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -84,7 +84,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_parallel_more_workers" "container_test:true"
+  Describe "test_parallel_more_workers" "container_test:true" "coverage:true"
     TEST_NAME="test_parallel_more_workers"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -130,7 +130,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:false"
+  Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -173,7 +173,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false"
+  Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 1bf857f..9b35ae6 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -19,7 +19,7 @@ Describe 'rollback'
   SUITE_NAME="rollback"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_rollback_due_to_timeout" "container_test:false"
+  Describe "test_rollback_due_to_timeout" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_due_to_timeout"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -67,7 +67,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_rollback_failed" "container_test:false"
+  Describe "test_rollback_failed" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -118,7 +118,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_rollback_ROLLBACK_ON_FAILURE_false" "container_test:false"
+  Describe "test_rollback_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -167,7 +167,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_rollback_lable_due_to_timeout" "container_test:false"
+  Describe "test_rollback_lable_due_to_timeout" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_lable_due_to_timeout"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -218,7 +218,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_rollback_label_failed" "container_test:false"
+  Describe "test_rollback_label_failed" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_label_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -274,7 +274,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_rollback_label_ROLLBACK_ON_FAILURE_false" "container_test:false"
+  Describe "test_rollback_label_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_label_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index 9e0b17f..cf22365 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -19,7 +19,7 @@ Describe 'service-multiple-services'
   SUITE_NAME="service-multiple-services"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_multiple_services_excluded_filters" "container_test:true"
+  Describe "test_multiple_services_excluded_filters" "container_test:true" "coverage:true"
     TEST_NAME="test_multiple_services_excluded_filters"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index 461c6af..0d3bfd6 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -19,7 +19,7 @@ Describe "service-no-running-tasks"
   SUITE_NAME="service-no-running-tasks"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_no_running_tasks_replicated" "container_test:true"
+  Describe "test_no_running_tasks_replicated" "container_test:true" "coverage:true"
     # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
@@ -98,7 +98,7 @@ Describe "service-no-running-tasks"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_no_running_tasks_global" "container_test:true"
+  Describe "test_no_running_tasks_global" "container_test:true" "coverage:true"
     TEST_NAME="test_no_running_tasks_global"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index edc5c20..b060d0f 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -19,7 +19,7 @@ Describe 'service-single-service'
   SUITE_NAME="service-single-service"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_new_image_no" "container_test:true"
+  Describe "test_new_image_no" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_no"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -64,7 +64,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
-  Describe "test_new_image_yes" "container_test:true"
+  Describe "test_new_image_yes" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_yes"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -109,7 +109,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
-  Describe "test_new_image_no_digest" "container_test:true"
+  Describe "test_new_image_no_digest" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_no_digest"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 0b091d2..036cb2b 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -19,7 +19,7 @@ Describe 'update-options'
   SUITE_NAME="update-options"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_update_jobs_skipping" "container_test:true"
+  Describe "test_update_jobs_skipping" "container_test:true" "coverage:true"
     # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
@@ -85,7 +85,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true"
+  Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
     TEST_NAME="test_update_jobs_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -132,7 +132,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true"
+  Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
     TEST_NAME="test_update_jobs_label_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -185,7 +185,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_jobs_no_running_tasks" "container_test:true"
+  Describe "test_update_jobs_no_running_tasks" "container_test:true" "coverage:true"
     TEST_NAME="test_update_jobs_no_running_tasks"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -242,7 +242,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_UPDATE_OPTIONS" "container_test:true"
+  Describe "test_update_UPDATE_OPTIONS" "container_test:true" "coverage:true"
     TEST_NAME="test_update_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -302,7 +302,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_label_UPDATE_OPTIONS" "container_test:true"
+  Describe "test_update_label_UPDATE_OPTIONS" "container_test:true" "coverage:true"
     TEST_NAME="test_update_label_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
@@ -365,7 +365,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false"
+  Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"

From 67301eee5fc40a4a10ee7080a3387a454c3c2d78 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 19 Nov 2024 22:18:56 -0800
Subject: [PATCH 08/27] [tests] add --update-monitor=1s to
 GANTRY_UPDATE_OPTIONS to reduce test runtime.

---
 tests/gantry_cleanup_images_spec.sh           | 1 +
 tests/gantry_common_options_spec.sh           | 1 +
 tests/gantry_filters_spec.sh                  | 1 +
 tests/gantry_login_docker_config_spec.sh      | 1 +
 tests/gantry_login_negative_spec.sh           | 1 +
 tests/gantry_login_spec.sh                    | 1 +
 tests/gantry_manifest_spec.sh                 | 1 +
 tests/gantry_notify_spec.sh                   | 1 +
 tests/gantry_parallel_spec.sh                 | 1 +
 tests/gantry_service_multiple_spec.sh         | 1 +
 tests/gantry_service_no_running_tasks_spec.sh | 1 +
 tests/gantry_update_options_spec.sh           | 1 +
 tests/spec_gantry_test_helper.sh              | 8 +++++++-
 13 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index 9613f4b..f0bfe37 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -18,6 +18,7 @@
 # Test cleanup images related options.
 Describe 'cleanup-images'
   SUITE_NAME="cleanup-images"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_CLEANUP_IMAGES_false" "container_test:true" "coverage:true"
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index bb3e7f2..4d94619 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -17,6 +17,7 @@
 
 Describe 'common-options'
   SUITE_NAME="common-options"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true"
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index 819cd4b..f046ce9 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -17,6 +17,7 @@
 
 Describe 'filters'
   SUITE_NAME="filters"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index 64c0714..74e4d02 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -33,6 +33,7 @@
 
 Describe 'login_docker_config'
   SUITE_NAME="login_docker_config"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   Describe "test_login_docker_config_no_label" "container_test:true" "coverage:true"
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index 809ec64..df52d89 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -17,6 +17,7 @@
 
 Describe 'login_negative'
   SUITE_NAME="login_negative"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   Describe "test_login_no_login" "container_test:false" "coverage:true"
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 11cea48..103dfe5 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -20,6 +20,7 @@ export IMAGE_DIGEST_WARNING="image .* could not be accessed on a registry to rec
 
 Describe 'login'
   SUITE_NAME="login"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   Describe "test_login_config" "container_test:true" "coverage:true"
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index ada408a..39788df 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -17,6 +17,7 @@
 
 Describe 'manifest-command'
   SUITE_NAME="manifest-command"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_MANIFEST_CMD_none" "container_test:true" "coverage:true"
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index b3fd52f..bdc9088 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -68,6 +68,7 @@ _print_and_cleanup_emails() {
 
 Describe 'notify'
   SUITE_NAME="notify"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "_notify_before_all ${SUITE_NAME}"
   AfterAll "_notify_after_all ${SUITE_NAME}"
   Describe "test_notify_apprise" "container_test:true" "coverage:true"
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index 34b7e4c..d9d2d4e 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -17,6 +17,7 @@
 
 Describe 'service-parallel'
   SUITE_NAME="service-parallel"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_parallel_less_workers" "container_test:true" "coverage:true"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index cf22365..5d56960 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -17,6 +17,7 @@
 
 Describe 'service-multiple-services'
   SUITE_NAME="service-multiple-services"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_multiple_services_excluded_filters" "container_test:true" "coverage:true"
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index 0d3bfd6..6e6ca66 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -17,6 +17,7 @@
 
 Describe "service-no-running-tasks"
   SUITE_NAME="service-no-running-tasks"
+  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_no_running_tasks_replicated" "container_test:true" "coverage:true"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 036cb2b..5b76aff 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -203,6 +203,7 @@ Describe 'update-options'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
+      export GANTRY_UPDATE_OPTIONS=
       export GANTRY_UPDATE_JOBS="true"
       run_gantry "${TEST_NAME}"
     }
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index ffdbb53..43658f0 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -419,7 +419,13 @@ reset_gantry_env() {
   export GANTRY_ROLLBACK_OPTIONS=
   export GANTRY_UPDATE_JOBS=
   export GANTRY_UPDATE_NUM_WORKERS=
-  export GANTRY_UPDATE_OPTIONS=
+  # Add --update-monitor to reduce the monitoring time to speedup tests. We could save about 4s on each service.
+  # Add --detach save more time, but we might hit problems about removing images.
+  if [ -n "${GANTRY_TEST_UPDATE_OPTIONS}" ]; then
+    export GANTRY_UPDATE_OPTIONS="${GANTRY_TEST_UPDATE_OPTIONS}"
+  else
+    export GANTRY_UPDATE_OPTIONS=
+  fi
   export GANTRY_UPDATE_TIMEOUT_SECONDS=
   export GANTRY_CLEANUP_IMAGES=
   export GANTRY_CLEANUP_IMAGES_OPTIONS=

From c8b961853ffa8fcf03782e7e1df27eed3b35002d Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 19 Nov 2024 23:08:51 -0800
Subject: [PATCH 09/27] [lib-common] Allow to use pre-run and post-run commands
 to change the gantry global configurations.

---
 src/lib-common.sh                   | 19 +++++++++++--------
 tests/gantry_common_options_spec.sh | 28 +++++++++++++++++-----------
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/src/lib-common.sh b/src/lib-common.sh
index 813f9b0..7297fba 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -295,19 +295,22 @@ eval_cmd() {
   local OLD_LOG_SCOPE="${LOG_SCOPE}"
   LOG_SCOPE=$(attach_tag_to_log_scope "${TAG}")
   export LOG_SCOPE
-  local LOG=
-  local RETURN_VALUE=0
   log INFO "Run ${TAG} command: ${CMD}"
-  if LOG=$(eval "${CMD}"); then
-    echo "${LOG}" | log_lines INFO
+  local EVAL_OUTPUT=
+  local EVAL_RETURN=
+  EVAL_OUTPUT=$(mktemp)
+  eval "${CMD}" > "${EVAL_OUTPUT}"
+  EVAL_RETURN=$?
+  if [ "${EVAL_RETURN}" = "0" ]; then
+    log_lines INFO < "${EVAL_OUTPUT}"
   else
-    RETURN_VALUE=$?
-    echo "${LOG}" | log_lines WARN
-    log WARN "${TAG} command returned a non-zero value ${RETURN_VALUE}."
+    log_lines WARN < "${EVAL_OUTPUT}"
+    log WARN "${TAG} command returned a non-zero value ${EVAL_RETURN}."
   fi
+  rm "${EVAL_OUTPUT}"
   log INFO "Finish ${TAG} command."
   export LOG_SCOPE="${OLD_LOG_SCOPE}"
-  return "${RETURN_VALUE}"
+  return "${EVAL_RETURN}"
 }
 
 swarm_network_arguments() {
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index 4d94619..380270f 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -112,16 +112,15 @@ Describe 'common-options'
       # There should be no warnings or errors. So it should work the same as LOG_LEVLE=NONE.
       export GANTRY_LOG_LEVEL=WARN
 
+      # Allow the following 3 mismatches used in log() function.
+      unset LOG_LEVEL NODE_NAME LOG_SCOPE
       # _ contains the last command. declare is a bash builtin.
       unset _; declare -p > "${ENV_BEFORE_RUN}"
       run_gantry "${TEST_NAME}"
-
       # Allow the following 3 mismatches used in log() function.
-      unset LOG_LEVEL
-      unset NODE_NAME
-      unset LOG_SCOPE
+      unset LOG_LEVEL NODE_NAME LOG_SCOPE
       unset _; declare -p > "${ENV_AFTER_RUN}"
-      diff <(cat "${ENV_BEFORE_RUN}") <(cat "${ENV_AFTER_RUN}")
+      diff "${ENV_BEFORE_RUN}" "${ENV_AFTER_RUN}"
       rm "${ENV_BEFORE_RUN}"
       rm "${ENV_AFTER_RUN}"
     }
@@ -144,8 +143,12 @@ Describe 'common-options'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_PRE_RUN_CMD="echo \"Pre update\""
-      export GANTRY_POST_RUN_CMD="echo \"Post update\""
+      export GANTRY_UPDATE_OPTIONS=
+      export GANTRY_CLEANUP_IMAGES=
+      # Test that pre-run command can change the global configurations.
+      export GANTRY_PRE_RUN_CMD="echo \"Pre update\"; GANTRY_UPDATE_OPTIONS=--detach=true; GANTRY_CLEANUP_IMAGES=false;"
+      # Test that the command returns a non-zero value.
+      export GANTRY_POST_RUN_CMD="echo \"Post update\"; false;"
       run_gantry "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -158,12 +161,14 @@ Describe 'common-options'
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "Pre update"
+      The stderr should satisfy spec_expect_no_message "pre-run command returned a non-zero value"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -174,11 +179,12 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
       The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
       The stderr should satisfy spec_expect_message    "Post update"
+      The stderr should satisfy spec_expect_message    "post-run command returned a non-zero value"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End

From 64ee7d62f69df414c1379574b2cc7cba6591cad4 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Wed, 20 Nov 2024 00:01:19 -0800
Subject: [PATCH 10/27] [gantry][tests] cleanup, add time saving options when
 creating the service.

diable check of START_WITHOUT_A_SQUARE_BRACKET on test_common_SLEEP_SECONDS.
report negative docker hub usage correctly.
add test_new_image_SERVICES_SELF.
---
 src/entrypoint.sh                             |   2 +-
 src/lib-common.sh                             |  15 ++-
 src/lib-gantry.sh                             | 103 +++++++++---------
 tests/gantry_cleanup_images_spec.sh           |   7 +-
 tests/gantry_common_options_spec.sh           |  10 +-
 tests/gantry_filters_spec.sh                  |   9 +-
 tests/gantry_login_docker_config_spec.sh      |   7 +-
 tests/gantry_login_negative_spec.sh           |  17 ++-
 tests/gantry_login_spec.sh                    |  12 +-
 tests/gantry_manifest_spec.sh                 |  13 +--
 tests/gantry_notify_spec.sh                   |  68 +++++-------
 tests/gantry_parallel_spec.sh                 |   9 +-
 tests/gantry_rollback_spec.sh                 |  12 +-
 tests/gantry_service_multiple_spec.sh         |   3 +-
 tests/gantry_service_no_running_tasks_spec.sh |   5 +-
 tests/gantry_service_single_spec.sh           |  55 +++++++++-
 tests/gantry_update_options_spec.sh           |  15 ++-
 tests/spec_gantry_test_helper.sh              |  42 ++++---
 18 files changed, 219 insertions(+), 185 deletions(-)

diff --git a/src/entrypoint.sh b/src/entrypoint.sh
index 38e3b11..e97caff 100755
--- a/src/entrypoint.sh
+++ b/src/entrypoint.sh
@@ -145,7 +145,7 @@ gantry() {
   local DOCKER_HUB_RATE_AFTER=
   local DOCKER_HUB_RATE_USED=
   DOCKER_HUB_RATE_AFTER=$(_read_docker_hub_rate)
-  DOCKER_HUB_RATE_USED=$(difference_between "${DOCKER_HUB_RATE_BEFORE}" "${DOCKER_HUB_RATE_AFTER}")
+  DOCKER_HUB_RATE_USED=$(first_minus_second "${DOCKER_HUB_RATE_BEFORE}" "${DOCKER_HUB_RATE_AFTER}")
   log INFO "After updating, Docker Hub rate remains ${DOCKER_HUB_RATE_AFTER}. Used rate ${DOCKER_HUB_RATE_USED}."
 
   gantry_finalize "${STACK}" "${ACCUMULATED_ERRORS}";
diff --git a/src/lib-common.sh b/src/lib-common.sh
index 7297fba..ed27c28 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -211,15 +211,11 @@ is_true() {
   echo "${CONFIG}" | grep_q_i "true"
 }
 
-difference_between() {
+first_minus_second() {
   local NUM0="${1}"
   local NUM1="${2}"
   if is_number "${NUM0}" && is_number "${NUM1}"; then
-    if [ "${NUM0}" -gt "${NUM1}" ]; then
-      echo "$((NUM0 - NUM1))"
-    else
-      echo "$((NUM1 - NUM0))"
-    fi
+    echo "$((NUM0 - NUM1))"
     return 0
   fi
   echo "NaN"
@@ -230,11 +226,14 @@ _time_elapsed_between() {
   local TIME0="${1}"
   local TIME1="${2}"
   local SECONDS_ELAPSED=
-  if ! SECONDS_ELAPSED=$(difference_between "${TIME0}" "${TIME1}"); then
+  if ! SECONDS_ELAPSED=$(first_minus_second "${TIME0}" "${TIME1}"); then
     echo "NaN"
     return 1
   fi
-  date -u -d "@${SECONDS_ELAPSED}" +'%-Mm %-Ss'
+  local FORMAT="%-Ss"
+  [ "${SECONDS_ELAPSED}" -ge 60 ] && FORMAT="%-Mm %-Ss"
+  [ "${SECONDS_ELAPSED}" -ge 3600 ] && FORMAT="%-Hh %-Mm %-Ss"
+  date -u -d "@${SECONDS_ELAPSED}" +"${FORMAT}"
 }
 
 time_elapsed_since() {
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index 382db2e..fb7faa9 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -33,10 +33,7 @@ _read_env_default() {
 gantry_read_number() {
   local ENV_NAME="${1}"
   local DEFAULT_VALUE="${2}"
-  if ! is_number "${DEFAULT_VALUE}"; then
-    log ERROR "DEFAULT_VALUE must be a number. Got \"${DEFAULT_VALUE}\"."
-    return 1
-  fi
+  ! is_number "${DEFAULT_VALUE}" && log ERROR "DEFAULT_VALUE must be a number. Got \"${DEFAULT_VALUE}\"." && return 1
   local VALUE=
   VALUE=$(_read_env_default "${ENV_NAME}" "${DEFAULT_VALUE}")
   if ! is_number "${VALUE}"; then
@@ -85,19 +82,31 @@ _get_docker_default_config() {
   readlink -f "${DEFAULT_LOCATION}"
 }
 
-# Record that the default config is used.
-_use_docker_default_config() {
-  local CONFIG_TO_REPORT=
-  CONFIG_TO_REPORT=$(_get_docker_default_config)
-  _static_variable_add_unique_to_list STATIC_VAR_DOCKER_CONFIG_DEFAULT "${CONFIG_TO_REPORT}"
+# Record that the default config is used when the input is either
+# 1. an empty string.
+# 2. same as _get_docker_default_config().
+_check_if_it_is_docker_default_config() {
+  local CONFIG_TO_CHECK="${1}"
+  local DEFAULT_LOCATION=
+  DEFAULT_LOCATION=$(_get_docker_default_config)
+  if [ -z "${CONFIG_TO_CHECK}" ]; then
+    CONFIG_TO_CHECK="${DEFAULT_LOCATION}"
+  else
+    CONFIG_TO_CHECK="$(readlink -f "${CONFIG_TO_CHECK}")"
+  fi
+  if [ "${CONFIG_TO_CHECK}" = "${DEFAULT_LOCATION}" ]; then
+    _static_variable_add_unique_to_list STATIC_VAR_DOCKER_CONFIG_DEFAULT "${DEFAULT_LOCATION}"
+  fi
 }
 
+# Echo the default docker config if it is used.
 # Return 0 when the default config is used
 # Return 1 when the default config is not used
 _docker_default_config_is_used() {
   local DOCKER_CONFIG_DEFAULT=
   DOCKER_CONFIG_DEFAULT=$(_static_variable_read_list STATIC_VAR_DOCKER_CONFIG_DEFAULT)
-  test -n "${DOCKER_CONFIG_DEFAULT}"
+  [ -z "${DOCKER_CONFIG_DEFAULT}" ] && return 1
+  echo "${DOCKER_CONFIG_DEFAULT}"
 }
 
 _login_registry() {
@@ -133,11 +142,7 @@ _login_registry() {
   if [ -n "${CONFIG}" ]; then
     _static_variable_add_unique_to_list STATIC_VAR_DOCKER_CONFIGS "${CONFIG}"
   fi
-  local DEFAULT_LOCATION=
-  DEFAULT_LOCATION=$(_get_docker_default_config)
-  if [ -z "${CONFIG}" ] || [ "$(readlink -f "${CONFIG}")" = "${DEFAULT_LOCATION}" ]; then
-    _use_docker_default_config
-  fi
+  _check_if_it_is_docker_default_config "${CONFIG}"
   return 0
 }
 
@@ -220,8 +225,8 @@ _authenticate_to_registries() {
     _login_registry "${USER}" "${PASSWORD}" "${HOST}" "${CONFIG}"
     ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))
   done < <(cat "${CONFIGS_FILE}"; echo;)
-  [ "${ACCUMULATED_ERRORS}" -gt 0 ] && return 1
-  return 0
+  # Do not simple return ACCUMULATED_ERRORS, in case it is larger than 255.
+  test "${ACCUMULATED_ERRORS}" = "0"
 }
 
 _send_notification() {
@@ -265,7 +270,6 @@ _remove_static_variables_folder() {
   local TO_REMOVE_STATIC_VARIABLES_FOLDER=
   TO_REMOVE_STATIC_VARIABLES_FOLDER="$(_get_static_variables_folder)"
   log DEBUG "Removing STATIC_VARIABLES_FOLDER ${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
-  export STATIC_VARIABLES_FOLDER=
   unset STATIC_VARIABLES_FOLDER
   rm -r "${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
 }
@@ -512,7 +516,7 @@ _report_services() {
   local STACK="${1:-gantry}"
   # ACCUMULATED_ERRORS is the number of errors that are not caused by updating.
   local ACCUMULATED_ERRORS="${2:-0}"
-  if ! is_number "${ACCUMULATED_ERRORS}"; then log WARN "ACCUMULATED_ERRORS \"${ACCUMULATED_ERRORS}\" is not a number." && ACCUMULATED_ERRORS=0; fi
+  ! is_number "${ACCUMULATED_ERRORS}" && log WARN "ACCUMULATED_ERRORS \"${ACCUMULATED_ERRORS}\" is not a number." && ACCUMULATED_ERRORS=0;
 
   local UPDATED_MSG=
   UPDATED_MSG=$(_report_services_from_static_variable STATIC_VAR_SERVICES_UPDATED "" "updated" "No services updated.")
@@ -692,8 +696,9 @@ _check_auth_config_folder() {
   log WARN "${AUTH_CONFIG} is not a directory that contains Docker configuration files."
   local MSG="configuration(s) set via GANTRY_REGISTRY_CONFIG or GANTRY_REGISTRY_CONFIGS_FILE"
   _report_from_static_variable STATIC_VAR_DOCKER_CONFIGS "There are" "${MSG}" "There are no ${MSG}." | log_lines WARN
-  if _docker_default_config_is_used; then
-    log WARN "User logged in using the default Docker configuration $(_get_docker_default_config)."
+  local DOCKER_CONFIG_DEFAULT=
+  if DOCKER_CONFIG_DEFAULT=$(_docker_default_config_is_used); then
+    log WARN "User logged in using the default Docker configuration ${DOCKER_CONFIG_DEFAULT}."
   fi
   return 1
 }
@@ -876,14 +881,13 @@ _get_number_of_running_tasks() {
 
 _get_with_registry_auth() {
   local AUTH_CONFIG="${1}"
-  local ARGUMENTS=""
   # AUTH_CONFIG is currently (2024.11) only used by Authentication.
   # When login is required, we must add `--with-registry-auth`. Otherwise the service will get an image without digest.
   # See https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336
-  if [ -n "${AUTH_CONFIG}" ] || _docker_default_config_is_used; then
-    ARGUMENTS="--with-registry-auth";
+  local DOCKER_CONFIG_DEFAULT=
+  if [ -n "${AUTH_CONFIG}" ] || DOCKER_CONFIG_DEFAULT=$(_docker_default_config_is_used); then
+    echo "--with-registry-auth";
   fi
-  echo "${ARGUMENTS}"
 }
 
 _get_service_update_additional_options() {
@@ -891,27 +895,28 @@ _get_service_update_additional_options() {
   local AUTH_CONFIG="${2}"
   local NUM_RUNS=
   NUM_RUNS=$(_get_number_of_running_tasks "${SERVICE_NAME}")
-  if ! is_number "${NUM_RUNS}"; then
-    log WARN "NUM_RUNS \"${NUM_RUNS}\" is not a number."
-    return 1
-  fi
-  local OPTIONS=
+  ! is_number "${NUM_RUNS}" && log WARN "NUM_RUNS \"${NUM_RUNS}\" is not a number." && return 1
+  local OPTIONS=""
+  local SPACE=""
   if [ "${NUM_RUNS}" = "0" ]; then
     # Add "--detach=true" when there is no running tasks.
     # https://github.com/docker/cli/issues/627
-    OPTIONS="${OPTIONS} --detach=true"
+    OPTIONS="${OPTIONS}${SPACE}--detach=true"
+    SPACE=" "
     local MODE=
     # Do not start a new task. Only works for replicated, not global.
     if MODE=$(_service_is_replicated "${SERVICE_NAME}"); then
-      OPTIONS="${OPTIONS} --replicas=0"
+      OPTIONS="${OPTIONS}${SPACE}--replicas=0"
+      SPACE=" "
     fi
   fi
   # Add `--with-registry-auth` if needed.
   local WITH_REGISTRY_AUTH=
   WITH_REGISTRY_AUTH="$(_get_with_registry_auth "${AUTH_CONFIG}")"
-  local SPACE=" "
-  [ -z "${OPTIONS}" ] && SPACE=""
-  [ -n "${WITH_REGISTRY_AUTH}" ] && OPTIONS="${OPTIONS}${SPACE}${WITH_REGISTRY_AUTH}"
+  if [ -n "${WITH_REGISTRY_AUTH}" ]; then
+    OPTIONS="${OPTIONS}${SPACE}${WITH_REGISTRY_AUTH}"
+    SPACE=" "
+  fi
   echo "${OPTIONS}"
 }
 
@@ -936,15 +941,15 @@ _rollback_service() {
   fi
   log INFO "Rolling back ${SERVICE_NAME}."
   # "service update --rollback" needs to take different options from "service update"
-  local ADDITIONAL_OPTIONS=
-  ADDITIONAL_OPTIONS=$(_get_service_rollback_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
-  [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
-  [ -n "${ROLLBACK_OPTIONS}" ] && log DEBUG "Adding options \"${ROLLBACK_OPTIONS}\" to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
+  local AUTOMATIC_OPTIONS=
+  AUTOMATIC_OPTIONS=$(_get_service_rollback_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
+  [ -n "${AUTOMATIC_OPTIONS}" ] && log DEBUG "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
+  [ -n "${ROLLBACK_OPTIONS}" ] && log DEBUG "Adding options \"${ROLLBACK_OPTIONS}\" specified by user to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
   local ROLLBACK_MSG=
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! ROLLBACK_MSG=$(docker ${AUTH_CONFIG} service update --quiet ${ADDITIONAL_OPTIONS} ${ROLLBACK_OPTIONS} --rollback "${SERVICE_NAME}" 2>&1); then
+  if ! ROLLBACK_MSG=$(docker ${AUTH_CONFIG} service update --quiet ${AUTOMATIC_OPTIONS} ${ROLLBACK_OPTIONS} --rollback "${SERVICE_NAME}" 2>&1); then
     log ERROR "Failed to roll back ${SERVICE_NAME}. ${ROLLBACK_MSG}"
     return 1
   fi
@@ -990,12 +995,12 @@ _update_single_service() {
   START_TIME=$(date +%s)
   log INFO "Updating ${SERVICE_NAME} with image ${IMAGE}"
   local AUTH_CONFIG=
-  local ADDITIONAL_OPTIONS=
+  local AUTOMATIC_OPTIONS=
   AUTH_CONFIG=$(_get_auth_config_from_service "${SERVICE_NAME}")
-  ADDITIONAL_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
+  AUTOMATIC_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
   [ -n "${AUTH_CONFIG}" ] && log DEBUG "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
-  [ -n "${ADDITIONAL_OPTIONS}" ] && log DEBUG "Adding options \"${ADDITIONAL_OPTIONS}\" to the command \"docker service update\" for ${SERVICE_NAME}."
-  [ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" to the command \"docker service update\" for ${SERVICE_NAME}."
+  [ -n "${AUTOMATIC_OPTIONS}" ] && log DEBUG "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update\" for ${SERVICE_NAME}."
+  [ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" specified by user to the command \"docker service update\" for ${SERVICE_NAME}."
   local TIMEOUT_COMMAND=""
   TIMEOUT_COMMAND=$(_get_timeout_command "${SERVICE_NAME}") || return 1
   local UPDATE_COMMAND="${TIMEOUT_COMMAND} docker ${AUTH_CONFIG} service update"
@@ -1004,7 +1009,7 @@ _update_single_service() {
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  UPDATE_MSG=$(${UPDATE_COMMAND} --quiet ${ADDITIONAL_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}" 2>&1);
+  UPDATE_MSG=$(${UPDATE_COMMAND} --quiet ${AUTOMATIC_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}" 2>&1);
   UPDATE_RETURN_VALUE=$?
   if [ "${UPDATE_RETURN_VALUE}" != 0 ]; then
     # https://git.savannah.gnu.org/cgit/coreutils.git/tree/src/timeout.c
@@ -1188,12 +1193,8 @@ gantry_finalize() {
   local STACK="${1:-gantry}"
   local NUM_ERRORS="${2:-0}"
   local RETURN_VALUE=0
-  if ! _remove_images "${STACK}_image-remover"; then
-    RETURN_VALUE=1
-  fi
-  if ! _report_services "${STACK}" "${NUM_ERRORS}"; then
-    RETURN_VALUE=1
-  fi
+  ! _remove_images "${STACK}_image-remover" && RETURN_VALUE=1
+  ! _report_services "${STACK}" "${NUM_ERRORS}" && RETURN_VALUE=1
   _remove_static_variables_folder
   return "${RETURN_VALUE}"
 }
diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index f0bfe37..e1cc202 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -18,7 +18,6 @@
 # Test cleanup images related options.
 Describe 'cleanup-images'
   SUITE_NAME="cleanup-images"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_CLEANUP_IMAGES_false" "container_test:true" "coverage:true"
@@ -40,7 +39,7 @@ Describe 'cleanup-images'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -84,7 +83,7 @@ Describe 'cleanup-images'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -129,7 +128,7 @@ Describe 'cleanup-images'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index 380270f..8672ae2 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -17,7 +17,6 @@
 
 Describe 'common-options'
   SUITE_NAME="common-options"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true"
@@ -43,7 +42,7 @@ Describe 'common-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_NOT_SWARM_MANAGER}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
@@ -159,7 +158,7 @@ Describe 'common-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "Pre update"
       The stderr should satisfy spec_expect_no_message "pre-run command returned a non-zero value"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -214,7 +213,8 @@ Describe 'common-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      # Do not check START_WITHOUT_A_SQUARE_BRACKET because the kill command could cause a "Broken pipe" error.
+      # The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_multiple_messages "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
       The stderr should satisfy spec_expect_no_message        "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message        "${NUM_SERVICES_SKIP_JOBS}"
@@ -260,7 +260,7 @@ Describe 'common-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "GANTRY_SLEEP_SECONDS ${MUST_BE_A_NUMBER}.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index f046ce9..68ba5d1 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -17,7 +17,6 @@
 
 Describe 'filters'
   SUITE_NAME="filters"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
@@ -39,7 +38,7 @@ Describe 'filters'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "Failed to obtain services list.*"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -96,7 +95,7 @@ Describe 'filters'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}"
       The stderr should satisfy spec_expect_multiple_messages "${EXCLUDE_SERVICE}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -147,7 +146,7 @@ Describe 'filters'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING_ALL}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
@@ -190,7 +189,7 @@ Describe 'filters'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "Failed to obtain services list.*"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index 74e4d02..0082bb1 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -33,7 +33,6 @@
 
 Describe 'login_docker_config'
   SUITE_NAME="login_docker_config"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   Describe "test_login_docker_config_no_label" "container_test:true" "coverage:true"
@@ -74,7 +73,7 @@ Describe 'login_docker_config'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
@@ -146,7 +145,7 @@ Describe 'login_docker_config'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
@@ -245,7 +244,7 @@ Describe 'login_docker_config'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index df52d89..d95421e 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -17,7 +17,6 @@
 
 Describe 'login_negative'
   SUITE_NAME="login_negative"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   Describe "test_login_no_login" "container_test:false" "coverage:true"
@@ -40,7 +39,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
@@ -109,7 +108,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
@@ -179,7 +178,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
@@ -257,7 +256,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
@@ -334,7 +333,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
@@ -447,7 +446,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME0}"
@@ -522,7 +521,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "format error.*Found extra item\(s\)"
       The stderr should satisfy spec_expect_message    "format error.*Missing item\(s\)"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
@@ -589,7 +588,7 @@ Describe 'login_negative'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 103dfe5..5296b21 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -20,7 +20,6 @@ export IMAGE_DIGEST_WARNING="image .* could not be accessed on a registry to rec
 
 Describe 'login'
   SUITE_NAME="login"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   Describe "test_login_config" "container_test:true" "coverage:true"
@@ -64,7 +63,7 @@ Describe 'login'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
@@ -77,7 +76,8 @@ Describe 'login'
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}"
       # 2 "--with-registry-auth" for SERVICE_NAME. One is automatically added. The other is from user.
-      The stderr should satisfy spec_expect_multiple_messages "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*automatically.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*specified by user.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
@@ -134,7 +134,7 @@ Describe 'login'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
@@ -207,7 +207,7 @@ Describe 'login'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
@@ -278,7 +278,7 @@ Describe 'login'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index 39788df..5793cb7 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -17,7 +17,6 @@
 
 Describe 'manifest-command'
   SUITE_NAME="manifest-command"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_MANIFEST_CMD_none" "container_test:true" "coverage:true"
@@ -40,7 +39,7 @@ Describe 'manifest-command'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       # Do not set GANTRY_SERVICES_SELF, it should be set autoamtically
       # If we are not testing gantry inside a container, it should failed to find the service name.
       # To test gantry container, we need to use run_gantry_container.
@@ -92,7 +91,7 @@ Describe 'manifest-command'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message ".*GANTRY_SERVICES_SELF.*"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
@@ -137,7 +136,7 @@ Describe 'manifest-command'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--insecure.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
@@ -186,7 +185,7 @@ Describe 'manifest-command'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--insecure.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
@@ -230,7 +229,7 @@ Describe 'manifest-command'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       # No options are added to the unknwon command.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_message    "Unknown MANIFEST_CMD.*unsupported_cmd"
@@ -285,7 +284,7 @@ Describe 'manifest-command'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_message    "Image.*${IMAGE_WITH_TAG}.*${IMAGE_NOT_EXIST}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index bdc9088..d4a8558 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -23,19 +23,25 @@ export SKIP_SENDING_NOTIFICATION="Skip sending notification."
 
 UINQUE_ID="$(unique_id)"
 export SERVICE_NAME_APPRISE="gantry-test-${UINQUE_ID}-apprise"
+export SERVICE_NAME_MAILPIT="gantry-test-${UINQUE_ID}-mailpit"
 # APPRISE_PORT is hard coded in the Apprise container.
 export APPRISE_PORT=8000
 export SMTP_PORT=1025
+export EMAIL_API_PORT=8025
 
 _notify_before_all() {
   local SUITE_NAME="${1}"
   initialize_all_tests "${SUITE_NAME}"
-  docker pull caronc/apprise
-  docker pull axllent/mailpit
-    # Use docker_run to improve coverage on lib-common.sh. `docker run` can do the same thing.
+  pull_image_if_not_exist caronc/apprise
+  pull_image_if_not_exist axllent/mailpit
+  # Use docker_run to improve coverage on lib-common.sh. `docker run` can do the same thing.
   docker_run -d --restart=on-failure:10 --name="${SERVICE_NAME_APPRISE}" --network=host \
     -e "APPRISE_STATELESS_URLS=mailto://localhost:${SMTP_PORT}?user=userid&pass=password" \
     caronc/apprise
+  docker_run -d --restart=on-failure:10 --name="${SERVICE_NAME_MAILPIT}" --network=host \
+    axllent/mailpit \
+    --smtp "localhost:${SMTP_PORT}" --listen "localhost:${EMAIL_API_PORT}" \
+    --smtp-auth-accept-any --smtp-auth-allow-insecure
 }
 
 _notify_after_all() {
@@ -43,32 +49,22 @@ _notify_after_all() {
   echo "Print Apprise log:"
   docker logs "${SERVICE_NAME_APPRISE}" 2>&1
   docker_remove "${SERVICE_NAME_APPRISE}" 2>&1
+  echo "Print Mailpit log:"
+  docker logs "${SERVICE_NAME_MAILPIT}" 2>&1
+  docker_remove "${SERVICE_NAME_MAILPIT}" 2>&1
   finish_all_tests "${SUITE_NAME}"
 }
 
-_setup_emails() {
-  local SERVICE_NAME_MAILPIT="${1}"
-  local EMAIL_API_PORT="${2}"
-  docker_run -d --restart=on-failure:10 --name="${SERVICE_NAME_MAILPIT}" --network=host \
-    axllent/mailpit \
-    --smtp "localhost:${SMTP_PORT}" --listen "localhost:${EMAIL_API_PORT}" \
-    --smtp-auth-accept-any --smtp-auth-allow-insecure
-}
-
 _print_and_cleanup_emails() {
-  local SERVICE_NAME_MAILPIT="${1}"
-  local EMAIL_API_PORT="${2}"
+  local API_URL="localhost:${EMAIL_API_PORT}/api/v1"
   echo "Print emails:"
-  curl --silent "localhost:${EMAIL_API_PORT}/api/v1/messages" 2>&1
-  echo "Print Mailpit log:"
-  docker logs "${SERVICE_NAME_MAILPIT}" 2>&1
-  # Use docker_remove to improve coverage on lib-common.sh. `docker stop` and `docker rm` can do the same thing.
-  docker_remove "${SERVICE_NAME_MAILPIT}" 2>&1
+  curl --silent "${API_URL}/messages" 2>&1
+  # Delete all messages
+  curl --silent -X "DELETE" "${API_URL}/messages" 2>&1
 }
 
 Describe 'notify'
   SUITE_NAME="notify"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "_notify_before_all ${SUITE_NAME}"
   AfterAll "_notify_after_all ${SUITE_NAME}"
   Describe "test_notify_apprise" "container_test:true" "coverage:true"
@@ -78,16 +74,13 @@ Describe 'notify'
     test_notify_apprise() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      local SERVICE_NAME_MAILPIT="${SERVICE_NAME}-mailpit"
       local RETURN_VALUE=0
-      local EMAIL_API_PORT=8025
-      _setup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
       reset_gantry_env "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      _print_and_cleanup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
+      _print_and_cleanup_emails
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -99,7 +92,7 @@ Describe 'notify'
       The stdout should satisfy spec_expect_message    "Subject.*1 services updated 0 failed TEST_TITLE"
       The stdout should satisfy spec_expect_message    "${TOTAL_EMAIL_COUNT_IS_ONE}"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -131,16 +124,13 @@ Describe 'notify'
     test_notify_apprise_no_new_image() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      local SERVICE_NAME_MAILPIT="${SERVICE_NAME}-mailpit"
       local RETURN_VALUE=0
-      local EMAIL_API_PORT=8025
-      _setup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
       reset_gantry_env "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      _print_and_cleanup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
+      _print_and_cleanup_emails
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -152,7 +142,7 @@ Describe 'notify'
       The stdout should satisfy spec_expect_message    "Subject.*0 services updated 0 failed TEST_TITLE"
       The stdout should satisfy spec_expect_message    "${TOTAL_EMAIL_COUNT_IS_ONE}"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -197,7 +187,7 @@ Describe 'notify'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -229,17 +219,14 @@ Describe 'notify'
     test_notify_on_change_new_image() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      local SERVICE_NAME_MAILPIT="${SERVICE_NAME}-mailpit"
       local RETURN_VALUE=0
-      local EMAIL_API_PORT=8025
-      _setup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
       reset_gantry_env "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_CONDITION="on-change"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      _print_and_cleanup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
+      _print_and_cleanup_emails
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -251,7 +238,7 @@ Describe 'notify'
       The stdout should satisfy spec_expect_message    "Subject.*1 services updated 0 failed TEST_TITLE"
       The stdout should satisfy spec_expect_message    "${TOTAL_EMAIL_COUNT_IS_ONE}"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -300,7 +287,7 @@ Describe 'notify'
       The stdout should satisfy spec_expect_no_message ".+"
       The stdout should satisfy spec_expect_no_message "TEST_TITLE"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -333,10 +320,7 @@ Describe 'notify'
     test_notify_on_change_errors() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      local SERVICE_NAME_MAILPIT="${SERVICE_NAME}-mailpit"
       local RETURN_VALUE=0
-      local EMAIL_API_PORT=8025
-      _setup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
       reset_gantry_env "${SERVICE_NAME}"
       export GANTRY_UPDATE_OPTIONS="--bad-options-that-causes-error"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
@@ -344,7 +328,7 @@ Describe 'notify'
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      _print_and_cleanup_emails "${SERVICE_NAME_MAILPIT}" "${EMAIL_API_PORT}"
+      _print_and_cleanup_emails
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -356,7 +340,7 @@ Describe 'notify'
       The stdout should satisfy spec_expect_message    "Subject.*0 services updated 1 failed TEST_TITLE"
       The stdout should satisfy spec_expect_message    "${TOTAL_EMAIL_COUNT_IS_ONE}"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index d9d2d4e..e0a5c03 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -17,7 +17,6 @@
 
 Describe 'service-parallel'
   SUITE_NAME="service-parallel"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_parallel_less_workers" "container_test:true" "coverage:true"
@@ -55,7 +54,7 @@ Describe 'service-parallel'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do
         SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
         The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} "
@@ -108,7 +107,7 @@ Describe 'service-parallel'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do
         SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
         The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.* ${SERVICE_NAME_NUM} "
@@ -150,7 +149,7 @@ Describe 'service-parallel'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "GANTRY_MANIFEST_NUM_WORKERS ${MUST_BE_A_NUMBER}.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
@@ -193,7 +192,7 @@ Describe 'service-parallel'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "GANTRY_UPDATE_NUM_WORKERS ${MUST_BE_A_NUMBER}.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 9b35ae6..3dea117 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -41,7 +41,7 @@ Describe 'rollback'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
@@ -92,7 +92,7 @@ Describe 'rollback'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
@@ -141,7 +141,7 @@ Describe 'rollback'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
@@ -192,7 +192,7 @@ Describe 'rollback'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
@@ -248,7 +248,7 @@ Describe 'rollback'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
@@ -301,7 +301,7 @@ Describe 'rollback'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index 5d56960..06c38bd 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -17,7 +17,6 @@
 
 Describe 'service-multiple-services'
   SUITE_NAME="service-multiple-services"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_multiple_services_excluded_filters" "container_test:true" "coverage:true"
@@ -68,7 +67,7 @@ Describe 'service-multiple-services'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       # Service 0 and 3 should get updated.
       # Service 1 and 2 should be excluded.
       # Service 4 and 5 created with new image, no update.
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index 6e6ca66..d46f3a4 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -17,7 +17,6 @@
 
 Describe "service-no-running-tasks"
   SUITE_NAME="service-no-running-tasks"
-  export GANTRY_TEST_UPDATE_OPTIONS="--update-monitor=1s"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
   Describe "test_no_running_tasks_replicated" "container_test:true" "coverage:true"
@@ -68,7 +67,7 @@ Describe "service-no-running-tasks"
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       # Add "--detach=true" when there is no running tasks.
       # https://github.com/docker/cli/issues/627
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}\."
@@ -130,7 +129,7 @@ Describe "service-no-running-tasks"
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       # Add "--detach=true" when there is no running tasks.
       # https://github.com/docker/cli/issues/627
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index b060d0f..a33a210 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -37,7 +37,7 @@ Describe 'service-single-service'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -82,7 +82,7 @@ Describe 'service-single-service'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -137,7 +137,7 @@ Describe 'service-single-service'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_DIGEST_IS_EMPTY}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -165,4 +165,53 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
+  Describe "test_new_image_SERVICES_SELF" "container_test:true" "coverage:true"
+    TEST_NAME="test_new_image_SERVICES_SELF"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    test_new_image_SERVICES_SELF() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SERVICE_NAME}"
+      # Explicitly set GANTRY_SERVICES_SELF
+      export GANTRY_SERVICES_SELF="${SERVICE_NAME}"
+      run_gantry "${TEST_NAME}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_new_image_SERVICES_SELF "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      # Gantry will inspect GANTRY_SERVICES_SELF then immediate update it.
+      # We won't see the NUM_SERVICES_UPDATING message because GANTRY_SERVICES_SELF is not added to the list.
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
+    End
+  End
 End # Describe 'Simple'
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 5b76aff..5fe0642 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -58,7 +58,7 @@ Describe 'update-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       # Check whether it is a job before checking whether there is a new image.
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME_SUFFIX}.*${SKIP_REASON_IS_JOB}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_SUFFIX}"
@@ -106,7 +106,7 @@ Describe 'update-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -159,7 +159,7 @@ Describe 'update-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -203,7 +203,6 @@ Describe 'update-options'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_UPDATE_OPTIONS=
       export GANTRY_UPDATE_JOBS="true"
       run_gantry "${TEST_NAME}"
     }
@@ -216,7 +215,7 @@ Describe 'update-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
       # Cannot add "--replicas" to replicated job
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0"
@@ -278,7 +277,7 @@ Describe 'update-options'
       The stdout should satisfy spec_expect_no_message "Before updating: LABEL_VALUE=.*${SERVICE_NAME}"
       The stdout should satisfy spec_expect_message    "After updating: LABEL_VALUE=.*${SERVICE_NAME}"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -341,7 +340,7 @@ Describe 'update-options'
       The stdout should satisfy spec_expect_no_message "Before updating: LABEL_VALUE=.*${SERVICE_NAME}"
       The stdout should satisfy spec_expect_message    "After updating: LABEL_VALUE=.*${SERVICE_NAME}"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -385,7 +384,7 @@ Describe 'update-options'
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${NOT_START_WITH_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "UPDATE_TIMEOUT_SECONDS ${MUST_BE_A_NUMBER}.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 43658f0..cccd49c 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -18,8 +18,8 @@
 set -a
 
 # Constant strings for checks.
-# NOT_START_WITH_A_SQUARE_BRACKET ignores color codes. Use test_log not to trigger this check.
-export NOT_START_WITH_A_SQUARE_BRACKET="^(?!(?:\x1b\[[0-9;]*[mG])?\[)"
+# START_WITHOUT_A_SQUARE_BRACKET ignores color codes. Use test_log not to trigger this check.
+export START_WITHOUT_A_SQUARE_BRACKET="^(?!(?:\x1b\[[0-9;]*[mG])?\[)"
 export GANTRY_AUTH_CONFIG_LABEL="gantry.auth.config"
 export MUST_BE_A_NUMBER="must be a number"
 export LOGGED_INTO_REGISTRY="Logged into registry"
@@ -407,9 +407,8 @@ reset_gantry_env() {
   export GANTRY_REGISTRY_USER_FILE=
   export GANTRY_SERVICES_EXCLUDED=
   export GANTRY_SERVICES_EXCLUDED_FILTERS=
-  if [ -z "${SERVICE_NAME}" ]; then
-    export GANTRY_SERVICES_FILTERS=
-  else
+  export GANTRY_SERVICES_FILTERS=
+  if [ -n "${SERVICE_NAME}" ]; then
     export GANTRY_SERVICES_FILTERS="name=${SERVICE_NAME}"
   fi
   export GANTRY_SERVICES_SELF=
@@ -419,13 +418,7 @@ reset_gantry_env() {
   export GANTRY_ROLLBACK_OPTIONS=
   export GANTRY_UPDATE_JOBS=
   export GANTRY_UPDATE_NUM_WORKERS=
-  # Add --update-monitor to reduce the monitoring time to speedup tests. We could save about 4s on each service.
-  # Add --detach save more time, but we might hit problems about removing images.
-  if [ -n "${GANTRY_TEST_UPDATE_OPTIONS}" ]; then
-    export GANTRY_UPDATE_OPTIONS="${GANTRY_TEST_UPDATE_OPTIONS}"
-  else
-    export GANTRY_UPDATE_OPTIONS=
-  fi
+  export GANTRY_UPDATE_OPTIONS=
   export GANTRY_UPDATE_TIMEOUT_SECONDS=
   export GANTRY_CLEANUP_IMAGES=
   export GANTRY_CLEANUP_IMAGES_OPTIONS=
@@ -627,6 +620,13 @@ _location_constraints() {
   echo "${ARGS}"
 }
 
+pull_image_if_not_exist() {
+  local IMAGE="${1}"
+  if ! docker image inspect "${IMAGE}" > /dev/null 2>&1; then
+    docker pull "${IMAGE}" > /dev/null
+  fi
+}
+
 _enforce_login_enabled() {
   local ENFORCE_LOGIN="${1}"
   test "${ENFORCE_LOGIN}" == "ENFORCE_LOGIN"
@@ -643,9 +643,7 @@ _add_htpasswd() {
   # https://distribution.github.io/distribution/about/deploying/#native-basic-auth
   local PASSWD=
   PASSWD="$(mktemp)"
-  if ! docker image inspect "${HTTPD_IMAGE}" > /dev/null 2>&1; then
-    docker pull "${HTTPD_IMAGE}" > /dev/null
-  fi
+  pull_image_if_not_exist "${HTTPD_IMAGE}"
   docker_run --entrypoint htpasswd "${HTTPD_IMAGE}" -Bbn "${USER}" "${PASS}" > "${PASSWD}"
   echo "--mount type=bind,source=${PASSWD},target=${PASSWD} \
         -e REGISTRY_AUTH=htpasswd \
@@ -673,7 +671,13 @@ start_replicated_service() {
   local IMAGE_WITH_TAG="${2}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "Creating service ${SERVICE_NAME} in replicated mode "
-  # Add --detach to reduce the test runtime.
+  # During creation:
+  # * Add --detach to reduce the test runtime.
+  # For updating:
+  # * Add --update-monitor=1s to save about 4s for each service update.
+  # * Add --stop-grace-period=1s to save about 4s for each service update.
+  # For rolling back:
+  # * Add --rollback-monitor=1s: needs to exam the effect.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
   timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
@@ -681,6 +685,9 @@ start_replicated_service() {
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
     --with-registry-auth \
+    --update-monitor=1s \
+    --stop-grace-period=1s \
+    --rollback-monitor=1s \
     $(_location_constraints) \
     --mode=replicated \
     --detach=true \
@@ -717,6 +724,9 @@ start_global_service() {
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
     --with-registry-auth \
+    --update-monitor=1s \
+    --stop-grace-period=1s \
+    --rollback-monitor=1s \
     $(_location_constraints) \
     --mode=global \
     "${IMAGE_WITH_TAG}"

From af6f475f8bcaf9e85436823a8cc9c955060159b8 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 04:49:46 -0800
Subject: [PATCH 11/27] [tests] create gantry_jobs.spec.sh, improve setup time
 of tests.

Try to reduce the conflict of ports.
---
 .github/workflows/on-pull-request.yml         |   1 +
 .github/workflows/on-push.yml                 |   1 +
 tests/gantry_cleanup_images_spec.sh           |   5 +-
 tests/gantry_common_options_spec.sh           |   4 +-
 tests/gantry_jobs_spec.sh                     | 245 +++++++++++++
 tests/gantry_rollback_spec.sh                 |  77 +----
 tests/gantry_service_no_running_tasks_spec.sh |   3 +-
 tests/gantry_update_options_spec.sh           | 327 +++++-------------
 tests/spec_gantry_test_helper.sh              |  38 +-
 9 files changed, 373 insertions(+), 328 deletions(-)
 create mode 100644 tests/gantry_jobs_spec.sh

diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml
index e9d0e88..2e4bab3 100644
--- a/.github/workflows/on-pull-request.yml
+++ b/.github/workflows/on-pull-request.yml
@@ -37,6 +37,7 @@ jobs:
           - gantry_cleanup_images_spec.sh
           - gantry_common_options_spec.sh
           - gantry_filters_spec.sh
+          - gantry_jobs_spec.sh
           - gantry_login_docker_config_spec.sh
           - gantry_login_negative_spec.sh
           - gantry_login_spec.sh
diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml
index 73a2097..127ceae 100644
--- a/.github/workflows/on-push.yml
+++ b/.github/workflows/on-push.yml
@@ -40,6 +40,7 @@ jobs:
           - gantry_cleanup_images_spec.sh
           - gantry_common_options_spec.sh
           - gantry_filters_spec.sh
+          - gantry_jobs_spec.sh
           - gantry_login_docker_config_spec.sh
           - gantry_login_negative_spec.sh
           - gantry_login_spec.sh
diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index e1cc202..2481aa4 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -173,12 +173,13 @@ Describe 'cleanup-images'
       local SERVICE_NAME0="${SERVICE_NAME}-0"
       local SERVICE_NAME1="${SERVICE_NAME}-1"
       local TASK_SECONDS=15
+      local TIMEOUT_SECONDS=1
       initialize_test "${TEST_NAME}"
       # The task will finish in ${TASK_SECONDS} seconds
       build_and_push_test_image "${IMAGE_WITH_TAG0}" "${TASK_SECONDS}"
-      start_global_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG0}"
+      start_global_service "${SERVICE_NAME0}" "${IMAGE_WITH_TAG0}" "${TIMEOUT_SECONDS}"
       build_and_push_test_image "${IMAGE_WITH_TAG1}"
-      start_global_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG1}"
+      start_global_service "${SERVICE_NAME1}" "${IMAGE_WITH_TAG1}" "${TIMEOUT_SECONDS}"
       # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks.
       wait_zero_running_tasks "${SERVICE_NAME0}"
       # Do not creat the Image IMAGE_WITH_TAG2, to run the test on a non-exist image.
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index 8672ae2..87a8407 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -199,10 +199,10 @@ Describe 'common-options'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_SLEEP_SECONDS="5"
+      export GANTRY_SLEEP_SECONDS="7"
       run_gantry "${TEST_NAME}" &
       local PID="${!}"
-      sleep 15
+      sleep $((GANTRY_SLEEP_SECONDS*2+1))
       kill "${PID}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_jobs_spec.sh b/tests/gantry_jobs_spec.sh
new file mode 100644
index 0000000..496780e
--- /dev/null
+++ b/tests/gantry_jobs_spec.sh
@@ -0,0 +1,245 @@
+#!/bin/bash spellspec
+# Copyright (C) 2024 Shizun Ge
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+Describe 'update-jobs'
+  SUITE_NAME="update-jobs"
+  BeforeAll "initialize_all_tests ${SUITE_NAME}"
+  AfterAll "finish_all_tests ${SUITE_NAME}"
+  Describe "test_update_jobs_skipping" "container_test:true" "coverage:true"
+    # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
+    # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
+    # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
+    # This test also checks whether we do an extra step to to perform the exact match.
+    TEST_NAME="test_update_jobs_skipping"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
+    test_start() {
+      local TEST_NAME="${1}"
+      local IMAGE_WITH_TAG="${2}"
+      local SERVICE_NAME="${3}"
+      local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
+      common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}"
+      start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
+    }
+    test_update_jobs_skipping() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SERVICE_NAME}"
+      run_gantry "${TEST_NAME}"
+    }
+    test_end() {
+      local TEST_NAME="${1}"
+      local IMAGE_WITH_TAG="${2}"
+      local SERVICE_NAME="${3}"
+      local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
+      stop_service "${SERVICE_NAME}"
+      common_cleanup "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}"
+    }
+    BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_update_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      # Check whether it is a job before checking whether there is a new image.
+      The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME_SUFFIX}.*${SKIP_REASON_IS_JOB}"
+      The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_SUFFIX}"
+      The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
+      The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+    TEST_NAME="test_update_jobs_UPDATE_JOBS_true"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    test_update_jobs_UPDATE_JOBS_true() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SERVICE_NAME}"
+      export GANTRY_UPDATE_JOBS="true"
+      # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true"
+      export GANTRY_UPDATE_OPTIONS="--detach=true"
+      run_gantry "${TEST_NAME}"
+    }
+    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_update_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+    TEST_NAME="test_update_jobs_label_UPDATE_JOBS_true"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    test_update_jobs_label_UPDATE_JOBS_true() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SERVICE_NAME}"
+      # label should override the global environment variable.
+      export GANTRY_UPDATE_JOBS="false"
+      local LABEL_AND_VALUE="gantry.update.jobs=true"
+      docker_service_update --detach=true --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
+      # label should override the global environment variable.
+      export GANTRY_UPDATE_OPTIONS="--incorrect-option"
+      # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true"
+      LABEL_AND_VALUE="gantry.update.options=--detach=true"
+      docker_service_update --detach=true --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
+      run_gantry "${TEST_NAME}"
+    }
+    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_update_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+  Describe "test_update_jobs_no_running_tasks" "container_test:true" "coverage:true"
+    TEST_NAME="test_update_jobs_no_running_tasks"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME="gantry-test-$(unique_id)"
+    TASK_SECONDS=1
+    test_start() {
+      local TEST_NAME="${1}"
+      local IMAGE_WITH_TAG="${2}"
+      local SERVICE_NAME="${3}"
+      local TASK_SECONDS="${4}"
+      common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME}" "${TASK_SECONDS}"
+      # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks.
+      wait_zero_running_tasks "${SERVICE_NAME}"
+    }
+    test_update_jobs_no_running_tasks() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SERVICE_NAME}"
+      export GANTRY_UPDATE_JOBS="true"
+      run_gantry "${TEST_NAME}"
+    }
+    # The task will finish in ${TASK_SECONDS} seconds
+    BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_update_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
+      # Cannot add "--replicas" to replicated job
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+    End
+  End
+End # Describe 'update-jobs'
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 3dea117..23d5b20 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -23,7 +23,7 @@ Describe 'rollback'
     TEST_NAME="test_rollback_due_to_timeout"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    TIMEOUT=3
+    TIMEOUT=1
     test_rollback_due_to_timeout() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -71,7 +71,7 @@ Describe 'rollback'
     TEST_NAME="test_rollback_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    TIMEOUT=3
+    TIMEOUT=1
     test_rollback_failed() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -122,7 +122,7 @@ Describe 'rollback'
     TEST_NAME="test_rollback_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    TIMEOUT=3
+    TIMEOUT=1
     test_rollback_ROLLBACK_ON_FAILURE_false() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -167,76 +167,23 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_rollback_lable_due_to_timeout" "container_test:false" "coverage:true"
-    TEST_NAME="test_rollback_lable_due_to_timeout"
-    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
-    TIMEOUT=3
-    test_rollback_lable_due_to_timeout() {
-      local TEST_NAME="${1}"
-      local SERVICE_NAME="${2}"
-      local TIMEOUT="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
-      # label should override the global environment variable.
-      export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
-      # Assume service update won't be done within TIMEOUT second.
-      local LABEL_AND_VALUE="gantry.update.timeout_seconds=${TIMEOUT}"
-      docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
-    }
-    BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}"
-    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    It 'run_test'
-      When run test_rollback_lable_due_to_timeout "${TEST_NAME}" "${SERVICE_NAME}" "${TIMEOUT}"
-      The status should be failure
-      The stdout should satisfy display_output
-      The stdout should satisfy spec_expect_no_message ".+"
-      The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
-      The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${ROLLED_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATE_FAILED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
-      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
-    End
-  End
   Describe "test_rollback_label_failed" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_label_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    TIMEOUT=3
+    TIMEOUT=1
     test_rollback_label_failed() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
       reset_gantry_env "${SERVICE_NAME}"
+      # Assume service update won't be done within TIMEOUT second.
+      export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
       # label should override the global environment variable.
-      export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
       export GANTRY_ROLLBACK_OPTIONS="--insecure"
-      # Assume service update won't be done within TIMEOUT second.
-      local LABEL_AND_VALUE="gantry.update.timeout_seconds=${TIMEOUT}"
-      docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       # Rollback would fail due to the incorrect option.
       # --with-registry-auth cannot be combined with --rollback.
-      LABEL_AND_VALUE="gantry.rollback.options=--with-registry-auth"
+      local LABEL_AND_VALUE="gantry.rollback.options=--with-registry-auth"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       run_gantry "${TEST_NAME}"
     }
@@ -278,18 +225,16 @@ Describe 'rollback'
     TEST_NAME="test_rollback_label_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    TIMEOUT=3
+    TIMEOUT=1
     test_rollback_label_ROLLBACK_ON_FAILURE_false() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
       reset_gantry_env "${SERVICE_NAME}"
-      # label should override the global environment variable.
-      export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
       # Assume service update won't be done within TIMEOUT second.
-      local LABEL_AND_VALUE="gantry.update.timeout_seconds=${TIMEOUT}"
-      docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      LABEL_AND_VALUE="gantry.rollback.on_failure=false"
+      export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
+      # label should override the global environment variable.
+      local LABEL_AND_VALUE="gantry.rollback.on_failure=false"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       run_gantry "${TEST_NAME}"
     }
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index d46f3a4..e210c85 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -107,10 +107,11 @@ Describe "service-no-running-tasks"
       local IMAGE_WITH_TAG="${2}"
       local SERVICE_NAME="${3}"
       local TASK_SECONDS=15
+      local TIMEOUT_SECONDS=1
       initialize_test "${TEST_NAME}"
       # The task will finish in ${TASK_SECONDS} seconds
       build_and_push_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}"
-      start_global_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
+      start_global_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${TIMEOUT_SECONDS}"
       build_and_push_test_image "${IMAGE_WITH_TAG}"
       # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks.
       wait_zero_running_tasks "${SERVICE_NAME}"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 5fe0642..e60768e 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -15,96 +15,45 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+_read_service_label() {
+  local SERVICE_NAME="${1}"
+  local LABEL="${2}"
+  docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}"
+}
+
 Describe 'update-options'
   SUITE_NAME="update-options"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_update_jobs_skipping" "container_test:true" "coverage:true"
-    # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
-    # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
-    # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
-    # This test also checks whether we do an extra step to to perform the exact match.
-    TEST_NAME="test_update_jobs_skipping"
-    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
-    SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
-    test_start() {
-      local TEST_NAME="${1}"
-      local IMAGE_WITH_TAG="${2}"
-      local SERVICE_NAME="${3}"
-      local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
-      common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}"
-      start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
-    }
-    test_update_jobs_skipping() {
-      local TEST_NAME="${1}"
-      local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
-    }
-    test_end() {
-      local TEST_NAME="${1}"
-      local IMAGE_WITH_TAG="${2}"
-      local SERVICE_NAME="${3}"
-      local SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
-      stop_service "${SERVICE_NAME}"
-      common_cleanup "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}"
-    }
-    BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    It 'run_test'
-      When run test_update_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}"
-      The status should be success
-      The stdout should satisfy display_output
-      The stdout should satisfy spec_expect_no_message ".+"
-      The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      # Check whether it is a job before checking whether there is a new image.
-      The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME_SUFFIX}.*${SKIP_REASON_IS_JOB}"
-      The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_SUFFIX}"
-      The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
-      The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_SKIP_JOBS}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_NO_NEW_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
-      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
-    End
-  End
-  Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_UPDATE_JOBS_true"
+  Describe "test_update_UPDATE_OPTIONS" "container_test:true" "coverage:true"
+    TEST_NAME="test_update_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    test_update_jobs_UPDATE_JOBS_true() {
+    test_update_UPDATE_OPTIONS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
+      local LABEL="gantry.test"
+      local LABEL_AND_VALUE=
+      LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
+      echo "Before updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
       reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_UPDATE_JOBS="true"
-      # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true"
-      export GANTRY_UPDATE_OPTIONS="--detach=true"
+      export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}"
+      local RETURN_VALUE=
       run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
+      echo "After updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
+      return "${RETURN_VALUE}"
     }
-    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_update_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
-      The stdout should satisfy spec_expect_no_message ".+"
+      # Check an observable difference before and after applying UPDATE_OPTIONS.
+      The stdout should satisfy spec_expect_no_message "Before updating: LABEL_AND_VALUE=.*${SERVICE_NAME}"
+      The stdout should satisfy spec_expect_message    "After updating: LABEL_AND_VALUE=.*${SERVICE_NAME}"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -114,7 +63,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -127,98 +76,44 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
       The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
-      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_label_UPDATE_JOBS_true"
+  Describe "test_update_label_UPDATE_OPTIONS" "container_test:true" "coverage:true"
+    TEST_NAME="test_update_label_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    test_update_jobs_label_UPDATE_JOBS_true() {
+    test_update_label_UPDATE_OPTIONS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
+      local LABEL="gantry.test"
+      local LABEL_AND_VALUE=
+      LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
+      echo "Before updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
       reset_gantry_env "${SERVICE_NAME}"
       # label should override the global environment variable.
-      export GANTRY_UPDATE_JOBS="false"
-      local LABEL_AND_VALUE="gantry.update.jobs=true"
-      docker_service_update --detach=true --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      # label should override the global environment variable.
       export GANTRY_UPDATE_OPTIONS="--incorrect-option"
-      # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true"
-      LABEL_AND_VALUE="gantry.update.options=--detach=true"
-      docker_service_update --detach=true --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
-    }
-    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    It 'run_test'
-      When run test_update_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
-      The status should be success
-      The stdout should satisfy display_output
-      The stdout should satisfy spec_expect_no_message ".+"
-      The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
-      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
-      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
-    End
-  End
-  Describe "test_update_jobs_no_running_tasks" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_no_running_tasks"
-    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
-    TASK_SECONDS=15
-    test_start() {
-      local TEST_NAME="${1}"
-      local IMAGE_WITH_TAG="${2}"
-      local SERVICE_NAME="${3}"
-      local TASK_SECONDS="${4}"
-      common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME}" "${TASK_SECONDS}"
-      # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks.
-      wait_zero_running_tasks "${SERVICE_NAME}"
-    }
-    test_update_jobs_no_running_tasks() {
-      local TEST_NAME="${1}"
-      local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_UPDATE_JOBS="true"
+      local LABEL_AND_VALUE="gantry.update.options=--label-add=${LABEL}=${SERVICE_NAME}"
+      docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
+      local RETURN_VALUE=
       run_gantry "${TEST_NAME}"
+      RETURN_VALUE="${?}"
+      LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
+      echo "After updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
+      return "${RETURN_VALUE}"
     }
-    # The task will finish in ${TASK_SECONDS} seconds
-    BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}"
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_update_label_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
-      The stdout should satisfy spec_expect_no_message ".+"
+      # Check an observable difference before and after applying UPDATE_OPTIONS.
+      The stdout should satisfy spec_expect_no_message "Before updating: LABEL_AND_VALUE=.*${SERVICE_NAME}"
+      The stdout should satisfy spec_expect_message    "After updating: LABEL_AND_VALUE=.*${SERVICE_NAME}"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
-      # Cannot add "--replicas" to replicated job
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -226,6 +121,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -242,42 +138,27 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_UPDATE_OPTIONS" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_UPDATE_OPTIONS"
+  Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true"
+    TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    _read_service_label() {
-      local SERVICE_NAME="${1}"
-      local LABEL="${2}"
-      docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}"
-    }
-    test_update_UPDATE_OPTIONS() {
+    test_update_UPDATE_TIMEOUT_SECONDS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      local LABEL="gantry.test"
-      local LABEL_VALUE=
-      LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
-      echo "Before updating: LABEL_VALUE=${LABEL_VALUE}"
       reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}"
-      local RETURN_VALUE=
+      export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
       run_gantry "${TEST_NAME}"
-      RETURN_VALUE="${?}"
-      LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
-      echo "After updating: LABEL_VALUE=${LABEL_VALUE}"
-      return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}"
-      The status should be success
+      When run test_update_UPDATE_TIMEOUT_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be failure
       The stdout should satisfy display_output
-      # Check an observable difference before and after applying UPDATE_OPTIONS.
-      The stdout should satisfy spec_expect_no_message "Before updating: LABEL_VALUE=.*${SERVICE_NAME}"
-      The stdout should satisfy spec_expect_message    "After updating: LABEL_VALUE=.*${SERVICE_NAME}"
+      The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_message    "UPDATE_TIMEOUT_SECONDS ${MUST_BE_A_NUMBER}.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
@@ -285,65 +166,63 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_message    "1 ${SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
-      The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_label_UPDATE_OPTIONS" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_label_UPDATE_OPTIONS"
+  Describe "test_update_lable_UPDATE_TIMEOUT_SECONDS" "container_test:true" "coverage:true"
+    TEST_NAME="test_update_lable_UPDATE_TIMEOUT_SECONDS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME="gantry-test-$(unique_id)"
-    _read_service_label() {
-      local SERVICE_NAME="${1}"
-      local LABEL="${2}"
-      docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}"
-    }
-    test_update_label_UPDATE_OPTIONS() {
+    TIMEOUT=300
+    test_update_lable_UPDATE_TIMEOUT_SECONDS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
+      local TIMEOUT="${3}"
       local LABEL="gantry.test"
-      local LABEL_VALUE=
-      LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
-      echo "Before updating: LABEL_VALUE=${LABEL_VALUE}"
+      local LABEL_AND_VALUE=
+      LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
+      echo "Before updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
       reset_gantry_env "${SERVICE_NAME}"
+      export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}"
       # label should override the global environment variable.
-      export GANTRY_UPDATE_OPTIONS="--incorrect-option"
-      local LABEL_AND_VALUE="gantry.update.options=--label-add=${LABEL}=${SERVICE_NAME}"
+      export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
+      # Assume that the update will finish within TIMEOUT.
+      LABEL_AND_VALUE="gantry.update.timeout_seconds=${TIMEOUT}"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       local RETURN_VALUE=
       run_gantry "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      LABEL_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
-      echo "After updating: LABEL_VALUE=${LABEL_VALUE}"
+      LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
+      echo "After updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_label_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_update_lable_UPDATE_TIMEOUT_SECONDS "${TEST_NAME}" "${SERVICE_NAME}" "${TIMEOUT}"
       The status should be success
       The stdout should satisfy display_output
       # Check an observable difference before and after applying UPDATE_OPTIONS.
-      The stdout should satisfy spec_expect_no_message "Before updating: LABEL_VALUE=.*${SERVICE_NAME}"
-      The stdout should satisfy spec_expect_message    "After updating: LABEL_VALUE=.*${SERVICE_NAME}"
+      The stdout should satisfy spec_expect_no_message "Before updating: LABEL_AND_VALUE=.*${SERVICE_NAME}"
+      The stdout should satisfy spec_expect_message    "After updating: LABEL_AND_VALUE=.*${SERVICE_NAME}"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
@@ -365,48 +244,4 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
     End
   End
-  Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true"
-    TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
-    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
-    test_update_UPDATE_TIMEOUT_SECONDS_not_a_number() {
-      local TEST_NAME="${1}"
-      local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
-      run_gantry "${TEST_NAME}"
-    }
-    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
-    It 'run_test'
-      When run test_update_UPDATE_TIMEOUT_SECONDS_not_a_number "${TEST_NAME}" "${SERVICE_NAME}"
-      The status should be failure
-      The stdout should satisfy display_output
-      The stdout should satisfy spec_expect_no_message ".+"
-      The stderr should satisfy display_output
-      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_message    "UPDATE_TIMEOUT_SECONDS ${MUST_BE_A_NUMBER}.*"
-      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
-      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
-      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_ERRORS}"
-      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
-      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
-      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
-    End
-  End
-End # Describe 'Single service'
+End # Describe 'update-options'
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index cccd49c..7832ab8 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -139,12 +139,14 @@ common_setup_timeout() {
   local IMAGE_WITH_TAG="${2}"
   local SERVICE_NAME="${3}"
   local TIMEOUT="${4}"
-  local DOUBLE_TIMEOUT=$((TIMEOUT+TIMEOUT))
+  local TIMEOUT_PLUS_ONE=$((TIMEOUT+1))
+  local TIMEOUT_PLUS_TWO=$((TIMEOUT+2))
   initialize_test "${TEST_NAME}"
   # -1 thus the task runs forever.
   # exit will take double of the timeout.
-  build_and_push_test_image "${IMAGE_WITH_TAG}" "-1" "${DOUBLE_TIMEOUT}"
-  start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
+  build_and_push_test_image "${IMAGE_WITH_TAG}" "-1" "${TIMEOUT_PLUS_TWO}"
+  # Timeout set by service create should be smaller than the exit time above.
+  start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${TIMEOUT_PLUS_ONE}"
   build_and_push_test_image "${IMAGE_WITH_TAG}"
 }
 
@@ -196,6 +198,13 @@ _get_sut_image() {
   echo "${SUT_REPO_TAG}"
 }
 
+_get_initial_port() {
+  # local BASE="${1}"
+  local PID=$$
+  local RANDOM_NUM=$(( PID % 1000 ))
+  echo $(( 55000 + RANDOM_NUM ))
+}
+
 _next_available_port() {
   local START="${1}"
   local LIMIT="${2:-1}"
@@ -290,11 +299,13 @@ load_test_registry() {
 _start_registry() {
   local SUITE_NAME="${1:?}"
   local ENFORCE_LOGIN="${2}"
+  local TIMEOUT_SECONDS="${3:-1}"
   SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-')
   local SUITE_NAME_LENGTH="${#SUITE_NAME}"
   local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}"
   local REGISTRY_BASE="127.0.0.1"
-  local REGISTRY_PORT=$((55000+SUITE_NAME_LENGTH*2))
+  local REGISTRY_PORT=
+  REGISTRY_PORT=$(_get_initial_port "${SUITE_NAME_LENGTH}")
   local TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}"
   export TEST_USERNAME="gantry"
   export TEST_PASSWORD="gantry"
@@ -321,6 +332,9 @@ _start_registry() {
       --name "${REGISTRY_SERVICE_NAME}" \
       --restart-condition "on-failure" \
       --restart-max-attempts 5 \
+      --update-monitor="${TIMEOUT_SECONDS}s" \
+      --stop-grace-period="${TIMEOUT_SECONDS}s" \
+      --rollback-monitor="${TIMEOUT_SECONDS}s" \
       $(_location_constraints) \
       --mode=replicated \
       -p "${REGISTRY_PORT}:5000" \
@@ -517,7 +531,7 @@ unique_id() {
   local RANDOM_STR=
   # repository name must be lowercase
   RANDOM_STR=$(head /dev/urandom | LANG=C tr -dc 'a-z0-9' | head -c 8)
-  echo "$(date +%s)-${PID}-${RANDOM_STR}"
+  echo "${PID}-$(date +%s)-${RANDOM_STR}"
 }
 
 build_test_image() {
@@ -669,6 +683,7 @@ _wait_service_state() {
 start_replicated_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
+  local TIMEOUT_SECONDS="${3:-1}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "Creating service ${SERVICE_NAME} in replicated mode "
   # During creation:
@@ -685,9 +700,9 @@ start_replicated_service() {
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
     --with-registry-auth \
-    --update-monitor=1s \
-    --stop-grace-period=1s \
-    --rollback-monitor=1s \
+    --update-monitor="${TIMEOUT_SECONDS}s" \
+    --stop-grace-period="${TIMEOUT_SECONDS}s" \
+    --rollback-monitor="${TIMEOUT_SECONDS}s" \
     $(_location_constraints) \
     --mode=replicated \
     --detach=true \
@@ -714,6 +729,7 @@ start_multiple_replicated_services() {
 start_global_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
+  local TIMEOUT_SECONDS="${3:-1}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "Creating service ${SERVICE_NAME} in global mode "
   # Do not add --detach, because we want to wait for the job finishes.
@@ -724,9 +740,9 @@ start_global_service() {
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
     --with-registry-auth \
-    --update-monitor=1s \
-    --stop-grace-period=1s \
-    --rollback-monitor=1s \
+    --update-monitor="${TIMEOUT_SECONDS}s" \
+    --stop-grace-period="${TIMEOUT_SECONDS}s" \
+    --rollback-monitor="${TIMEOUT_SECONDS}s" \
     $(_location_constraints) \
     --mode=global \
     "${IMAGE_WITH_TAG}"

From d891952d040a54646cd597e4ccfc78f3f71ecf19 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 06:44:02 -0800
Subject: [PATCH 12/27] [tests] add get_test_service_name and
 _correct_service_name

---
 tests/gantry_cleanup_images_spec.sh           |  8 ++--
 tests/gantry_common_options_spec.sh           | 14 +++---
 tests/gantry_filters_spec.sh                  |  8 ++--
 tests/gantry_jobs_spec.sh                     | 19 +++++---
 tests/gantry_login_docker_config_spec.sh      |  6 +--
 tests/gantry_login_negative_spec.sh           | 16 +++----
 tests/gantry_login_spec.sh                    |  8 ++--
 tests/gantry_manifest_spec.sh                 | 12 ++---
 tests/gantry_notify_spec.sh                   | 12 ++---
 tests/gantry_parallel_spec.sh                 |  8 ++--
 tests/gantry_rollback_spec.sh                 | 10 ++--
 tests/gantry_service_multiple_spec.sh         |  2 +-
 tests/gantry_service_no_running_tasks_spec.sh |  4 +-
 tests/gantry_service_single_spec.sh           |  8 ++--
 tests/gantry_update_options_spec.sh           |  8 ++--
 tests/spec_gantry_test_helper.sh              | 46 +++++++++++++++----
 16 files changed, 112 insertions(+), 77 deletions(-)

diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index 2481aa4..245b854 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -23,7 +23,7 @@ Describe 'cleanup-images'
   Describe "test_CLEANUP_IMAGES_false" "container_test:true" "coverage:true"
     TEST_NAME="test_CLEANUP_IMAGES_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_CLEANUP_IMAGES_false() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -65,7 +65,7 @@ Describe 'cleanup-images'
   Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true" "coverage:true"
     TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_CLEANUP_IMAGES_OPTIONS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -111,7 +111,7 @@ Describe 'cleanup-images'
   Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true" "coverage:true"
     TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_good"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_CLEANUP_IMAGES_OPTIONS_good() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -160,7 +160,7 @@ Describe 'cleanup-images'
     IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0"
     IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
     IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2"
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME0="${SERVICE_NAME}-0"
     SERVICE_NAME1="${SERVICE_NAME}-1"
     test_start() {
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index 87a8407..b6d0985 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -22,7 +22,7 @@ Describe 'common-options'
   Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true"
     TEST_NAME="test_common_DOCKER_HOST_not_swarm_manager"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_common_DOCKER_HOST_not_swarm_manager() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -71,7 +71,7 @@ Describe 'common-options'
   Describe "test_common_LOG_LEVEL_none" "container_test:true" "coverage:true"
     TEST_NAME="test_common_LOG_LEVEL_none"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_common_LOG_LEVEL_none() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -98,7 +98,7 @@ Describe 'common-options'
     # We don't need to run this test using containers because we check env on the host, while the container test set env inside the container.
     TEST_NAME="test_common_no_new_env"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_common_no_new_env() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -137,7 +137,7 @@ Describe 'common-options'
   Describe "test_common_PRE_POST_RUN_CMD" "container_test:true" "coverage:true"
     TEST_NAME="test_common_PRE_POST_RUN_CMD"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_common_PRE_POST_RUN_CMD() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -194,7 +194,7 @@ Describe 'common-options'
   Describe "test_common_SLEEP_SECONDS" "container_test:false" "coverage:true"
     TEST_NAME="test_common_SLEEP_SECONDS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_common_SLEEP_SECONDS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -202,7 +202,7 @@ Describe 'common-options'
       export GANTRY_SLEEP_SECONDS="7"
       run_gantry "${TEST_NAME}" &
       local PID="${!}"
-      sleep $((GANTRY_SLEEP_SECONDS*2+1))
+      sleep $((GANTRY_SLEEP_SECONDS*3+1))
       kill "${PID}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -244,7 +244,7 @@ Describe 'common-options'
   Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_common_SLEEP_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_common_SLEEP_SECONDS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index 68ba5d1..98be545 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -22,7 +22,7 @@ Describe 'filters'
   Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
     TEST_NAME="test_SERVICES_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_SERVICES_FILTERS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -66,7 +66,7 @@ Describe 'filters'
   Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_multiple_services"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     MAX_SERVICES_NUM=10
     test_SERVICES_EXCLUDED_multiple_services() {
       local TEST_NAME="${1}"
@@ -123,7 +123,7 @@ Describe 'filters'
   Describe "test_SERVICES_EXCLUDED_FILTERS_default" "container_test:true" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     MAX_SERVICES_NUM=10
     test_SERVICES_EXCLUDED_FILTERS_default() {
       local TEST_NAME="${1}"
@@ -173,7 +173,7 @@ Describe 'filters'
   Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_SERVICES_EXCLUDED_FILTERS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
diff --git a/tests/gantry_jobs_spec.sh b/tests/gantry_jobs_spec.sh
index 496780e..6bd7171 100644
--- a/tests/gantry_jobs_spec.sh
+++ b/tests/gantry_jobs_spec.sh
@@ -26,7 +26,7 @@ Describe 'update-jobs'
     # This test also checks whether we do an extra step to to perform the exact match.
     TEST_NAME="test_update_jobs_skipping"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
     test_start() {
       local TEST_NAME="${1}"
@@ -88,7 +88,10 @@ Describe 'update-jobs'
   Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
     TEST_NAME="test_update_jobs_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
+    TASK_SECONDS=-1
+    # Use a long EXIT_SECONDS, because we want the task keep running for a while after updating to trigger the image removing failure.
+    EXIT_SECONDS=30
     test_update_jobs_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -98,7 +101,7 @@ Describe 'update-jobs'
       export GANTRY_UPDATE_OPTIONS="--detach=true"
       run_gantry "${TEST_NAME}"
     }
-    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
       When run test_update_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
@@ -135,7 +138,10 @@ Describe 'update-jobs'
   Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
     TEST_NAME="test_update_jobs_label_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
+    TASK_SECONDS=-1
+    # Use a long EXIT_SECONDS, because we want the task keep running for a while after updating to trigger the image removing failure.
+    EXIT_SECONDS=30
     test_update_jobs_label_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -151,7 +157,7 @@ Describe 'update-jobs'
       docker_service_update --detach=true --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       run_gantry "${TEST_NAME}"
     }
-    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
       When run test_update_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
@@ -188,7 +194,8 @@ Describe 'update-jobs'
   Describe "test_update_jobs_no_running_tasks" "container_test:true" "coverage:true"
     TEST_NAME="test_update_jobs_no_running_tasks"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
+    # Use a short TASK_SECONDS, because we want the task finishes soon.
     TASK_SECONDS=1
     test_start() {
       local TEST_NAME="${1}"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index 0082bb1..6afa5cb 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -38,7 +38,7 @@ Describe 'login_docker_config'
   Describe "test_login_docker_config_no_label" "container_test:true" "coverage:true"
     TEST_NAME="test_login_docker_config_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_no_label() {
@@ -108,7 +108,7 @@ Describe 'login_docker_config'
   Describe "test_login_docker_config_default_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_docker_config_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_default_config() {
@@ -178,7 +178,7 @@ Describe 'login_docker_config'
   Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
     TEST_NAME="test_login_docker_config_label_override"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME0="${SERVICE_NAME}-0"
     SERVICE_NAME1="${SERVICE_NAME}-1"
     SERVICE_NAME2="${SERVICE_NAME}-2"
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index d95421e..a54e61a 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -22,7 +22,7 @@ Describe 'login_negative'
   Describe "test_login_no_login" "container_test:false" "coverage:true"
     TEST_NAME="test_login_no_login"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_no_login() {
@@ -72,7 +72,7 @@ Describe 'login_negative'
   Describe "test_login_incorrect_password" "container_test:false" "coverage:true"
     TEST_NAME="test_login_incorrect_password"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_incorrect_password() {
@@ -140,7 +140,7 @@ Describe 'login_negative'
   Describe "test_login_read_only_file" "container_test:false" "coverage:true"
     TEST_NAME="test_login_read_only_file"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_read_only_file() {
@@ -210,7 +210,7 @@ Describe 'login_negative'
   Describe "test_login_config_mismatch_default" "container_test:false" "coverage:true"
     TEST_NAME="test_login_config_mismatch_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config_mismatch_default() {
@@ -294,7 +294,7 @@ Describe 'login_negative'
   Describe "test_login_config_mismatch_no_default" "container_test:false" "coverage:true"
     TEST_NAME="test_login_config_mismatch_no_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config_mismatch_no_default() {
@@ -373,7 +373,7 @@ Describe 'login_negative'
     # To test https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
     TEST_NAME="test_login_multi_services_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0"
     IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
     SERVICE_NAME0="${SERVICE_NAME}-0"
@@ -486,7 +486,7 @@ Describe 'login_negative'
   Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" "coverage:true"
     TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_REGISTRY_CONFIGS_FILE_bad_format() {
@@ -555,7 +555,7 @@ Describe 'login_negative'
   Describe "test_login_file_not_exist" "container_test:false" "coverage:true"
     TEST_NAME="test_login_file_not_exist"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_file_not_exist() {
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 5296b21..1b5c0b4 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -25,7 +25,7 @@ Describe 'login'
   Describe "test_login_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config() {
@@ -98,7 +98,7 @@ Describe 'login'
   Describe "test_login_default_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="NotUsed"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_default_config() {
@@ -167,7 +167,7 @@ Describe 'login'
   Describe "test_login_REGISTRY_CONFIGS_FILE" "container_test:true" "coverage:true"
     TEST_NAME="test_login_REGISTRY_CONFIGS_FILE"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_REGISTRY_CONFIGS_FILE() {
@@ -241,7 +241,7 @@ Describe 'login'
   Describe "test_login_external_config" "container_test:true" "coverage:true"
     TEST_NAME="test_login_external_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     CONFIG="$(mktemp -d)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_external_config() {
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index 5793cb7..493b21e 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -22,7 +22,7 @@ Describe 'manifest-command'
   Describe "test_MANIFEST_CMD_none" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_none"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_MANIFEST_CMD_none() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -72,7 +72,7 @@ Describe 'manifest-command'
   Describe "test_MANIFEST_CMD_none_SERVICES_SELF" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_none_SERVICES_SELF"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_MANIFEST_CMD_none_SERVICES_SELF() {
       # If the service is self, it will always run manifest checking. Even if the CMD is set to none
       local TEST_NAME="${1}"
@@ -119,7 +119,7 @@ Describe 'manifest-command'
   Describe "test_MANIFEST_CMD_manifest" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_manifest"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_MANIFEST_CMD_manifest() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -163,7 +163,7 @@ Describe 'manifest-command'
   Describe "test_MANIFEST_CMD_label" "container_test:true" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_MANIFEST_CMD_label() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -212,7 +212,7 @@ Describe 'manifest-command'
   Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_unsupported_cmd"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_MANIFEST_CMD_unsupported_cmd() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -258,7 +258,7 @@ Describe 'manifest-command'
   Describe "test_MANIFEST_CMD_failure" "container_test:false" "coverage:true"
     TEST_NAME="test_MANIFEST_CMD_failure"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_start() {
       # This test assumes that the IMAGE_WITH_TAG does not exist on the registry.
       # get_image_with_tag should return an image with a unique tag.
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index d4a8558..7e837e2 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -70,7 +70,7 @@ Describe 'notify'
   Describe "test_notify_apprise" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_apprise"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_notify_apprise() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -120,7 +120,7 @@ Describe 'notify'
   Describe "test_notify_apprise_no_new_image" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_apprise_no_new_image"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_notify_apprise_no_new_image() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -171,7 +171,7 @@ Describe 'notify'
   Describe "test_notify_apprise_bad_url" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_apprise_bad_url"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_notify_apprise_bad_url() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -215,7 +215,7 @@ Describe 'notify'
   Describe "test_notify_on_change_new_image" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_on_change_new_image"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_notify_on_change_new_image() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -266,7 +266,7 @@ Describe 'notify'
   Describe "test_notify_on_change_no_updates" "container_test:true" "coverage:true"
     TEST_NAME="test_notify_on_change_no_updates"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_notify_on_change_no_updates() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -316,7 +316,7 @@ Describe 'notify'
   Describe "test_notify_on_change_errors" "container_test:false" "coverage:true"
     TEST_NAME="test_notify_on_change_errors"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_notify_on_change_errors() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index e0a5c03..4d4e467 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -22,7 +22,7 @@ Describe 'service-parallel'
   Describe "test_parallel_less_workers" "container_test:true" "coverage:true"
     TEST_NAME="test_parallel_less_workers"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     MANIFEST_NUM_WORKERS=2
     MAX_SERVICES_NUM=6
     MAX_NO_NEW_IMAGE=3
@@ -87,7 +87,7 @@ Describe 'service-parallel'
   Describe "test_parallel_more_workers" "container_test:true" "coverage:true"
     TEST_NAME="test_parallel_more_workers"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     MANIFEST_NUM_WORKERS=2
     MAX_SERVICES_NUM=10
     test_parallel_more_workers() {
@@ -133,7 +133,7 @@ Describe 'service-parallel'
   Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -176,7 +176,7 @@ Describe 'service-parallel'
   Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 23d5b20..256a355 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -22,7 +22,7 @@ Describe 'rollback'
   Describe "test_rollback_due_to_timeout" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_due_to_timeout"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TIMEOUT=1
     test_rollback_due_to_timeout() {
       local TEST_NAME="${1}"
@@ -70,7 +70,7 @@ Describe 'rollback'
   Describe "test_rollback_failed" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TIMEOUT=1
     test_rollback_failed() {
       local TEST_NAME="${1}"
@@ -121,7 +121,7 @@ Describe 'rollback'
   Describe "test_rollback_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TIMEOUT=1
     test_rollback_ROLLBACK_ON_FAILURE_false() {
       local TEST_NAME="${1}"
@@ -170,7 +170,7 @@ Describe 'rollback'
   Describe "test_rollback_label_failed" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_label_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TIMEOUT=1
     test_rollback_label_failed() {
       local TEST_NAME="${1}"
@@ -224,7 +224,7 @@ Describe 'rollback'
   Describe "test_rollback_label_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
     TEST_NAME="test_rollback_label_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TIMEOUT=1
     test_rollback_label_ROLLBACK_ON_FAILURE_false() {
       local TEST_NAME="${1}"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index 06c38bd..1da1dfa 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -22,7 +22,7 @@ Describe 'service-multiple-services'
   Describe "test_multiple_services_excluded_filters" "container_test:true" "coverage:true"
     TEST_NAME="test_multiple_services_excluded_filters"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME0="${SERVICE_NAME}-0"
     SERVICE_NAME1="${SERVICE_NAME}-1"
     SERVICE_NAME2="${SERVICE_NAME}-2"
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index e210c85..78ebc1e 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -26,7 +26,7 @@ Describe "service-no-running-tasks"
     # This test also checks whether we do an extra step to to perform the exact match.
     TEST_NAME="test_no_running_tasks_replicated"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
     test_start() {
       local TEST_NAME="${1}"
@@ -101,7 +101,7 @@ Describe "service-no-running-tasks"
   Describe "test_no_running_tasks_global" "container_test:true" "coverage:true"
     TEST_NAME="test_no_running_tasks_global"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_start() {
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index a33a210..c37fedf 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -22,7 +22,7 @@ Describe 'service-single-service'
   Describe "test_new_image_no" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_no"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_new_image_no() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -67,7 +67,7 @@ Describe 'service-single-service'
   Describe "test_new_image_yes" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_yes"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_new_image_yes() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -112,7 +112,7 @@ Describe 'service-single-service'
   Describe "test_new_image_no_digest" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_no_digest"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_start() {
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
@@ -168,7 +168,7 @@ Describe 'service-single-service'
   Describe "test_new_image_SERVICES_SELF" "container_test:true" "coverage:true"
     TEST_NAME="test_new_image_SERVICES_SELF"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_new_image_SERVICES_SELF() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index e60768e..91bb74a 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -28,7 +28,7 @@ Describe 'update-options'
   Describe "test_update_UPDATE_OPTIONS" "container_test:true" "coverage:true"
     TEST_NAME="test_update_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_update_UPDATE_OPTIONS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -83,7 +83,7 @@ Describe 'update-options'
   Describe "test_update_label_UPDATE_OPTIONS" "container_test:true" "coverage:true"
     TEST_NAME="test_update_label_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_update_label_UPDATE_OPTIONS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -141,7 +141,7 @@ Describe 'update-options'
   Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true"
     TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     test_update_UPDATE_TIMEOUT_SECONDS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
@@ -185,7 +185,7 @@ Describe 'update-options'
   Describe "test_update_lable_UPDATE_TIMEOUT_SECONDS" "container_test:true" "coverage:true"
     TEST_NAME="test_update_lable_UPDATE_TIMEOUT_SECONDS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
-    SERVICE_NAME="gantry-test-$(unique_id)"
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TIMEOUT=300
     test_update_lable_UPDATE_TIMEOUT_SECONDS() {
       local TEST_NAME="${1}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 7832ab8..5198269 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -127,10 +127,11 @@ common_setup_job() {
   local IMAGE_WITH_TAG="${2}"
   local SERVICE_NAME="${3}"
   local TASK_SECONDS="${4}"
+  local EXIT_SECONDS="${5}"
   initialize_test "${TEST_NAME}"
   # The task will finish in ${TASK_SECONDS} seconds, when ${TASK_SECONDS} is not empty
-  build_and_push_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}"
-  _start_replicated_job "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
+  build_and_push_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" "${EXIT_SECONDS}"
+  _start_replicated_job "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" "${EXIT_SECONDS}"
   build_and_push_test_image "${IMAGE_WITH_TAG}"
 }
 
@@ -143,9 +144,9 @@ common_setup_timeout() {
   local TIMEOUT_PLUS_TWO=$((TIMEOUT+2))
   initialize_test "${TEST_NAME}"
   # -1 thus the task runs forever.
-  # exit will take double of the timeout.
+  # exit will take longer than the timeout.
   build_and_push_test_image "${IMAGE_WITH_TAG}" "-1" "${TIMEOUT_PLUS_TWO}"
-  # Timeout set by service create should be smaller than the exit time above.
+  # Timeout set by "service create" should be smaller than the exit time above.
   start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${TIMEOUT_PLUS_ONE}"
   build_and_push_test_image "${IMAGE_WITH_TAG}"
 }
@@ -313,6 +314,7 @@ _start_registry() {
   local TRIES=0
   local MAX_RETRIES=50
   local PORT_LIMIT=500
+  pull_image_if_not_exist "${REGISTRY_IMAGE}"
   while true; do
     if ! REGISTRY_PORT=$(_next_available_port "${REGISTRY_PORT}" "${PORT_LIMIT}" 2>&1); then
       echo "_start_registry _next_available_port error: ${REGISTRY_PORT}" >&2
@@ -528,10 +530,13 @@ unique_id() {
   # Try to generate a unique id.
   # To reduce the possibility that tests run in parallel on the same machine affect each other.
   local PID="$$"
+  local TIME_STR=
+  TIME_STR=$(date +%s)
+  TIME_STR=$((TIME_STR % 10000))
   local RANDOM_STR=
   # repository name must be lowercase
   RANDOM_STR=$(head /dev/urandom | LANG=C tr -dc 'a-z0-9' | head -c 8)
-  echo "${PID}-$(date +%s)-${RANDOM_STR}"
+  echo "${PID}-${TIME_STR}-${RANDOM_STR}"
 }
 
 build_test_image() {
@@ -544,7 +549,7 @@ build_test_image() {
     # Finsih the job in the given time.
     TASK_CMD="sleep ${TASK_SECONDS};"
   fi
-  local EXIT_CMD="sleep 0;"
+  local EXIT_CMD="true;"
   if [ -n "${EXIT_SECONDS}" ] && [ "${EXIT_SECONDS}" -gt "0" ]; then
     EXIT_CMD="sleep ${EXIT_SECONDS};"
   fi
@@ -552,6 +557,7 @@ build_test_image() {
   FILE=$(mktemp)
   echo "FROM alpinelinux/docker-cli:latest" > "${FILE}"
   echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
+  pull_image_if_not_exist alpinelinux/docker-cli:latest
   echo "Building ${IMAGE_WITH_TAG} "
   timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
   rm "${FILE}"
@@ -680,11 +686,29 @@ _wait_service_state() {
   done
 }
 
+_correct_service_name() {
+  local SERVICE_NAME="${1}"
+  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
+  echo "${SERVICE_NAME}"
+}
+
+get_test_service_name() {
+  local TEST_NAME="${1}"
+  local TEST_NAME_SHORT="${TEST_NAME}"
+  # Max length is 63. Leave some spaces for suffix
+  [ "${#TEST_NAME}" -gt 30 ] && TEST_NAME_SHORT=${TEST_NAME:0-30}
+  local SERVICE_NAME=
+  SERVICE_NAME="g$(unique_id)-${TEST_NAME_SHORT}"
+  SERVICE_NAME=$(echo "${SERVICE_NAME}" | tr '[:upper:]' '[:lower:]')
+  SERVICE_NAME=$(echo "${SERVICE_NAME}" | tr '_' '-')
+  echo "${SERVICE_NAME}"
+}
+
 start_replicated_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
   local TIMEOUT_SECONDS="${3:-1}"
-  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
+  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in replicated mode "
   # During creation:
   # * Add --detach to reduce the test runtime.
@@ -707,6 +731,7 @@ start_replicated_service() {
     --mode=replicated \
     --detach=true \
     "${IMAGE_WITH_TAG}"
+  _wait_service_state "${SERVICE_NAME}" "Running"
 }
 
 start_multiple_replicated_services() {
@@ -730,7 +755,7 @@ start_global_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
   local TIMEOUT_SECONDS="${3:-1}"
-  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
+  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in global mode "
   # Do not add --detach, because we want to wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
@@ -751,7 +776,9 @@ start_global_service() {
 _start_replicated_job() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
-  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
+  local TASK_SECONDS="${3:-1}"
+  local EXIT_SECONDS="${4:-1}"
+  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in replicated job mode "
   # Always add --detach=true, do not wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
@@ -761,6 +788,7 @@ _start_replicated_job() {
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
     --with-registry-auth \
+    --stop-grace-period="${EXIT_SECONDS}s" \
     $(_location_constraints) \
     --mode=replicated-job \
     --detach=true \

From 6d0c63ac4e20f4accbfac268c9e621a5b4da57c8 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 17:41:54 -0800
Subject: [PATCH 13/27] [docker] fix docker hub rate reporting when rate is -1.

---
 src/docker_hub_rate.sh | 35 +++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/docker_hub_rate.sh b/src/docker_hub_rate.sh
index 0159b3c..070d153 100755
--- a/src/docker_hub_rate.sh
+++ b/src/docker_hub_rate.sh
@@ -15,20 +15,25 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+_curl_installed() {
+  curl --version 1>/dev/null 2>&1;
+}
+
 _docker_hub_rate_token() {
   local IMAGE="${1:-ratelimitpreview/test}"
   local USER_AND_PASS="${2}"
   local TOKEN_URL="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${IMAGE}:pull"
-  if curl --version 1>/dev/null 2>&1; then
-    if [ -n "${USER_AND_PASS}" ]; then
-      curl --silent --show-error --user "${USER_AND_PASS}" "${TOKEN_URL}"
-      return $?
-    fi
-    curl --silent --show-error "${TOKEN_URL}"
+  if ! _curl_installed; then
+    [ -n "${USER_AND_PASS}" ] && log WARN "Cannot read docker hub rate for the given user because curl is not available."
+    wget -qO- "${TOKEN_URL}"
+    return $?
+  fi
+  if [ -n "${USER_AND_PASS}" ]; then
+    curl --silent --show-error --user "${USER_AND_PASS}" "${TOKEN_URL}"
     return $?
   fi
-  [ -n "${USER_AND_PASS}" ] && log WARN "Cannot read docker hub rate for the given user because curl is not available."
-  wget -qO- "${TOKEN_URL}"
+  curl --silent --show-error "${TOKEN_URL}"
+  return $?
 }
 
 _docker_hub_rate_read_rate() {
@@ -37,13 +42,13 @@ _docker_hub_rate_read_rate() {
   [ -z "${TOKEN}" ] && echo "[EMPTY TOKEN ERROR]" && return 1
   local HEADER="Authorization: Bearer ${TOKEN}"
   local URL="https://registry-1.docker.io/v2/${IMAGE}/manifests/latest"
-  if curl --version 1>/dev/null 2>&1; then
-    curl --silent --show-error --head -H "${HEADER}" "${URL}" 2>&1
+  if ! _curl_installed; then
+    # Add `--spider`` implies that you want to send a HEAD request (as opposed to GET or POST).
+    # The `busybox wget` does not send a HEAD request, thus it will consume a docker hub rate.
+    wget -qS --spider --header="${HEADER}" -O /dev/null "${URL}" 2>&1
     return $?
   fi
-  # Add `--spider`` implies that you want to send a HEAD request (as opposed to GET or POST).
-  # The `busybox wget` does not send a HEAD request, thus it will consume a docker hub rate.
-  wget -qS --spider --header="${HEADER}" -O /dev/null "${URL}" 2>&1
+  curl --silent --show-error --head -H "${HEADER}" "${URL}" 2>&1
 }
 
 _docker_hub_echo_error() {
@@ -81,6 +86,7 @@ docker_hub_rate() {
   fi
   if ! RESPONSE=$(_docker_hub_rate_read_rate "${IMAGE}" "${TOKEN}" 2>&1); then
     if echo "${RESPONSE}" | grep -q "Too Many Requests" ; then
+      # This occurs when we send request not via the HEAD method, i.e. using busybox wget.
       echo "0"
       return 0
     fi
@@ -88,10 +94,11 @@ docker_hub_rate() {
     return 1
   fi
   local RATE=
-  RATE=$(echo "${RESPONSE}" | sed -n 's/.*ratelimit-remaining: \([0-9]*\).*/\1/p' )
+  RATE=$(echo "${RESPONSE}" | sed -n 's/.*ratelimit-remaining: \([-]\?[0-9]\+\);.*/\1/p' )
   if [ -z "${RATE}" ]; then
     _docker_hub_echo_error "PARSE RATE ERROR" "${RESPONSE}"
     return 1
   fi
+  [ "${RATE}" -lt 0 ] && RATE=0;
   echo "${RATE}"
 }

From 93e150157043a1c04b86addd1c138eb8036d7306 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 17:48:31 -0800
Subject: [PATCH 14/27] [gantry] make sure that image remover service name is
 shorter than 64.

update comments
---
 src/lib-gantry.sh                | 13 ++++++++++---
 tests/spec_gantry_test_helper.sh |  8 ++++----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index fb7faa9..ffc0c90 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -403,6 +403,13 @@ gantry_remove_images() {
   log INFO "Done removing images.";
 }
 
+_correct_service_name() {
+  local SERVICE_NAME="${1}"
+  SERVICE_NAME=$(echo "${SERVICE_NAME}" | tr ' ' '-')
+  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME="${SERVICE_NAME:0:63}"
+  echo "${SERVICE_NAME}"
+}
+
 _remove_images() {
   local CLEANUP_IMAGES="${GANTRY_CLEANUP_IMAGES:-"true"}"
   local CLEANUP_IMAGES_OPTIONS="${GANTRY_CLEANUP_IMAGES_OPTIONS:-""}"
@@ -413,7 +420,7 @@ _remove_images() {
     return 0
   fi
   local SERVICE_NAME="${1:-"gantry-image-remover"}"
-  SERVICE_NAME=$(echo "${SERVICE_NAME}" | tr ' ' '-')
+  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
   docker_service_remove "${SERVICE_NAME}"
   local IMAGES_TO_REMOVE=
   IMAGES_TO_REMOVE=$(_static_variable_read_list STATIC_VAR_IMAGES_TO_REMOVE)
@@ -816,7 +823,7 @@ _inspect_image() {
     # The image may not contain the digest for the following reasons:
     # 1. The image has not been push to or pulled from a V2 registry
     # 2. The image has been pulled from a V1 registry
-    # 3. The service has not been updated via Docker CLI, but via Docker API, i.e. via 3rd party tools.
+    # 3. The service is updated without --with-registry-auth when registry requests authentication.
     log DEBUG "Perform updating ${SERVICE_NAME} because DIGEST is empty in ${IMAGE_WITH_DIGEST}, assume there is a new image."
     echo "${IMAGE}"
     return 0
@@ -1193,7 +1200,7 @@ gantry_finalize() {
   local STACK="${1:-gantry}"
   local NUM_ERRORS="${2:-0}"
   local RETURN_VALUE=0
-  ! _remove_images "${STACK}_image-remover" && RETURN_VALUE=1
+  ! _remove_images "${STACK}-image-remover" && RETURN_VALUE=1
   ! _report_services "${STACK}" "${NUM_ERRORS}" && RETURN_VALUE=1
   _remove_static_variables_folder
   return "${RETURN_VALUE}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 5198269..d7240c9 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -686,7 +686,7 @@ _wait_service_state() {
   done
 }
 
-_correct_service_name() {
+_correct_test_service_name() {
   local SERVICE_NAME="${1}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "${SERVICE_NAME}"
@@ -708,7 +708,7 @@ start_replicated_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
   local TIMEOUT_SECONDS="${3:-1}"
-  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in replicated mode "
   # During creation:
   # * Add --detach to reduce the test runtime.
@@ -755,7 +755,7 @@ start_global_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
   local TIMEOUT_SECONDS="${3:-1}"
-  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in global mode "
   # Do not add --detach, because we want to wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
@@ -778,7 +778,7 @@ _start_replicated_job() {
   local IMAGE_WITH_TAG="${2}"
   local TASK_SECONDS="${3:-1}"
   local EXIT_SECONDS="${4:-1}"
-  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in replicated job mode "
   # Always add --detach=true, do not wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.

From 0846d20cefe5c1b8a8c80fae920efce113a2b7bf Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 18:01:07 -0800
Subject: [PATCH 15/27] [lib-common] calculate time elapsed more than a day.

---
 src/lib-common.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/lib-common.sh b/src/lib-common.sh
index ed27c28..d3b3753 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -230,10 +230,19 @@ _time_elapsed_between() {
     echo "NaN"
     return 1
   fi
-  local FORMAT="%-Ss"
-  [ "${SECONDS_ELAPSED}" -ge 60 ] && FORMAT="%-Mm %-Ss"
-  [ "${SECONDS_ELAPSED}" -ge 3600 ] && FORMAT="%-Hh %-Mm %-Ss"
-  date -u -d "@${SECONDS_ELAPSED}" +"${FORMAT}"
+  local HOUR=0
+  local MIN=0
+  local SEC=0
+  HOUR=$((SECONDS_ELAPSED / 3600))
+  local WITHIN_AN_HOUR=0
+  WITHIN_AN_HOUR=$((SECONDS_ELAPSED % 3600))
+  MIN=$((WITHIN_AN_HOUR / 60))
+  SEC=$((WITHIN_AN_HOUR % 60))
+  local TIME_STR=""
+  [ "${HOUR}" != "0" ] && TIME_STR="${HOUR}h "
+  if [ -n "${TIME_STR}" ] || [ "${MIN}" != "0" ]; then TIME_STR="${TIME_STR}${MIN}m "; fi
+  TIME_STR="${TIME_STR}${SEC}s"
+  echo "${TIME_STR}"
 }
 
 time_elapsed_since() {

From 2d36b365bd2be5c0e68e5dd176990694cef812e6 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 19:38:10 -0800
Subject: [PATCH 16/27] [gantry][tests] fix flaky tests running in parallel.

---
 src/lib-gantry.sh | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index ffc0c90..cd7cf60 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -239,6 +239,18 @@ _send_notification() {
   notify_summary "${TYPE}" "${TITLE}" "${BODY}"
 }
 
+_get_static_variables_folder_name() {
+  local PID=$$
+  echo "/tmp/gantry-static-variables-folder-${PID}"
+}
+
+_make_static_variables_folder() {
+  STATIC_VARIABLES_FOLDER=$(_get_static_variables_folder_name)
+  local OUTPUT=
+  ! OUTPUT=$(mkdir -p "${STATIC_VARIABLES_FOLDER}" 2>&1) && log ERROR "failed: mkdir -p ${STATIC_VARIABLES_FOLDER}: ${OUTPUT}" && return 1
+  echo "${STATIC_VARIABLES_FOLDER}"
+}
+
 # We want the static variables live longer than a function.
 # However if we call the function in a subprocess, which could be casued by
 # 1. pipe, e.g. echo "message" | my_function
@@ -246,29 +258,21 @@ _send_notification() {
 # and changing the static variables, the value won't go back to the parent process.
 # So here we use the file system to pass value between multiple processes.
 _get_static_variables_folder() {
-  local INDIRECT_FILE="/tmp/gantry-STATIC_VARIABLES_FOLDER"
-  if [ -z "${STATIC_VARIABLES_FOLDER}" ]; then
-    if [ -e "${INDIRECT_FILE}" ]; then
-      STATIC_VARIABLES_FOLDER=$(head -1 "${INDIRECT_FILE}")
-    fi
-  fi
   if [ -d "${STATIC_VARIABLES_FOLDER}" ]; then
     echo "${STATIC_VARIABLES_FOLDER}"
     return 0
   fi
-  while ! STATIC_VARIABLES_FOLDER=$(mktemp -d); do log ERROR "\"mktemp -d\" failed"; done
-  log DEBUG "Created STATIC_VARIABLES_FOLDER ${STATIC_VARIABLES_FOLDER}"
-  echo "${STATIC_VARIABLES_FOLDER}" > "${INDIRECT_FILE}"
+  log DEBUG "Creating STATIC_VARIABLES_FOLDER"
+  STATIC_VARIABLES_FOLDER=$(_make_static_variables_folder)
+  log DEBUG "Created STATIC_VARIABLES_FOLDER \"${STATIC_VARIABLES_FOLDER}\""
   export STATIC_VARIABLES_FOLDER
   echo "${STATIC_VARIABLES_FOLDER}"
 }
 
 _remove_static_variables_folder() {
-  local INDIRECT_FILE="/tmp/gantry-STATIC_VARIABLES_FOLDER"
-  rm "${INDIRECT_FILE}" >/dev/null 2>&1
   [ -z "${STATIC_VARIABLES_FOLDER}" ] && return 0
   local TO_REMOVE_STATIC_VARIABLES_FOLDER=
-  TO_REMOVE_STATIC_VARIABLES_FOLDER="$(_get_static_variables_folder)"
+  TO_REMOVE_STATIC_VARIABLES_FOLDER="$(_get_static_variables_folder_name)"
   log DEBUG "Removing STATIC_VARIABLES_FOLDER ${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
   unset STATIC_VARIABLES_FOLDER
   rm -r "${TO_REMOVE_STATIC_VARIABLES_FOLDER}"

From df29dc50fe9d7d0c207fcaa04514611ab9c203d2 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 21:16:40 -0800
Subject: [PATCH 17/27] [tests] use docker run to start registries

service create cmd consumes a docker hub rate.
---
 tests/gantry_notify_spec.sh      |  2 ++
 tests/spec_gantry_test_helper.sh | 50 ++++++++++++++++++++------------
 2 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index 7e837e2..d20ad15 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -34,6 +34,8 @@ _notify_before_all() {
   initialize_all_tests "${SUITE_NAME}"
   pull_image_if_not_exist caronc/apprise
   pull_image_if_not_exist axllent/mailpit
+  docker_remove "${SERVICE_NAME_APPRISE}" 1>/dev/null 2>&1
+  docker_remove "${SERVICE_NAME_MAILPIT}" 1>/dev/null 2>&1
   # Use docker_run to improve coverage on lib-common.sh. `docker run` can do the same thing.
   docker_run -d --restart=on-failure:10 --name="${SERVICE_NAME_APPRISE}" --network=host \
     -e "APPRISE_STATELESS_URLS=mailto://localhost:${SMTP_PORT}?user=userid&pass=password" \
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index d7240c9..14f3440 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -70,6 +70,8 @@ export FAILED_TO_REMOVE_IMAGE="Failed to remove image"
 export SCHEDULE_NEXT_UPDATE_AT="Schedule next update at"
 export SLEEP_SECONDS_BEFORE_NEXT_UPDATE="Sleep [0-9]+ seconds before next update"
 
+export TEST_IMAGE_REMOVER="ghcr.io/shizunge/gantry-development"
+
 test_log() {
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^NONE$"  && return 0;
   [ -n "${GANTRY_IMAGES_TO_REMOVE}" ] && echo "${*}" >&2 && return 0;
@@ -304,7 +306,7 @@ _start_registry() {
   SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-')
   local SUITE_NAME_LENGTH="${#SUITE_NAME}"
   local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}"
-  local REGISTRY_BASE="127.0.0.1"
+  local REGISTRY_BASE="localhost"
   local REGISTRY_PORT=
   REGISTRY_PORT=$(_get_initial_port "${SUITE_NAME_LENGTH}")
   local TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}"
@@ -324,36 +326,46 @@ _start_registry() {
       echo "_start_registry _next_available_port error: REGISTRY_PORT is empty." >&2
       return 1
     fi
-    stop_service "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1
+    docker container stop "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
+    docker container rm -f "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
     TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}"
     echo "Suite \"${SUITE_NAME}\" starts registry ${TEST_REGISTRY} "
+    local CID=
     # SC2046 (warning): Quote this to prevent word splitting.
     # SC2086 (info): Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2046,SC2086
-    if docker service create --quiet \
+    if CID=$(docker container run -d --rm \
       --name "${REGISTRY_SERVICE_NAME}" \
-      --restart-condition "on-failure" \
-      --restart-max-attempts 5 \
-      --update-monitor="${TIMEOUT_SECONDS}s" \
-      --stop-grace-period="${TIMEOUT_SECONDS}s" \
-      --rollback-monitor="${TIMEOUT_SECONDS}s" \
-      $(_location_constraints) \
-      --mode=replicated \
-      -p "${REGISTRY_PORT}:5000" \
+      --network=host \
+      -e "REGISTRY_HTTP_ADDR=${TEST_REGISTRY}" \
+      -e "REGISTRY_HTTP_HOST=http://${TEST_REGISTRY}" \
+      --stop-timeout "${TIMEOUT_SECONDS}" \
       $(_add_htpasswd "${ENFORCE_LOGIN}" "${TEST_USERNAME}" "${TEST_PASSWORD}") \
-      "${REGISTRY_IMAGE}" 2>&1; then
+      "${REGISTRY_IMAGE}" 2>&1); then
+      local STATUS=
+      while [ "${STATUS}" != "running" ]; do
+        STATUS=$(docker container inspect "${CID}" --format '{{.State.Status}}')
+      done
       break;
     fi
     if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
       echo "_start_registry Reach MAX_RETRIES ${MAX_RETRIES}" >&2
       return 1
     fi
-    TRIES=$((TRIES+1))
     REGISTRY_PORT=$((REGISTRY_PORT+1))
+    TRIES=$((TRIES+1))
     sleep 1
   done
   _store_test_registry "${SUITE_NAME}" "${TEST_REGISTRY}" || return 1;
-  _login_test_registry "${ENFORCE_LOGIN}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" || return 1;
+  TRIES=0
+  while ! _login_test_registry "${ENFORCE_LOGIN}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"; do
+    if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
+      echo "_login_test_registry Reach MAX_RETRIES ${MAX_RETRIES}" >&2
+      return 1
+    fi
+    TRIES=$((TRIES+1))
+    sleep 1
+  done
 }
 
 _stop_registry() {
@@ -364,7 +376,8 @@ _stop_registry() {
   local REGISTRY=
   REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
   echo "Removing registry ${REGISTRY} "
-  stop_service "${REGISTRY_SERVICE_NAME}"
+  docker container stop "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
+  docker container rm -f "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
   _logout_test_registry "${ENFORCE_LOGIN}" "${REGISTRY}" || return 1
   _remove_test_registry_file "${SUITE_NAME}" || return 1
   return 0
@@ -382,6 +395,7 @@ initialize_all_tests() {
   echo "=============================="
   _init_swarm
   _start_registry "${SUITE_NAME}" "${ENFORCE_LOGIN}"
+  pull_image_if_not_exist "${TEST_IMAGE_REMOVER}"
 }
 
 # finish_all_tests should return non zero when there are errors.
@@ -438,7 +452,7 @@ reset_gantry_env() {
   export GANTRY_UPDATE_TIMEOUT_SECONDS=
   export GANTRY_CLEANUP_IMAGES=
   export GANTRY_CLEANUP_IMAGES_OPTIONS=
-  export GANTRY_CLEANUP_IMAGES_REMOVER=ghcr.io/shizunge/gantry-development
+  export GANTRY_CLEANUP_IMAGES_REMOVER="${TEST_IMAGE_REMOVER}"
   export GANTRY_IMAGES_TO_REMOVE=
   export GANTRY_NOTIFICATION_APPRISE_URL=
   export GANTRY_NOTIFICATION_CONDITION=
@@ -557,8 +571,8 @@ build_test_image() {
   FILE=$(mktemp)
   echo "FROM alpinelinux/docker-cli:latest" > "${FILE}"
   echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
-  pull_image_if_not_exist alpinelinux/docker-cli:latest
-  echo "Building ${IMAGE_WITH_TAG} "
+  pull_image_if_not_exist "alpinelinux/docker-cli:latest"
+  echo "Building ${IMAGE_WITH_TAG} from ${FILE}"
   timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
   rm "${FILE}"
 }

From 291d0a59d238abf4c644e5284108ce8a6e33e1f1 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 21:27:54 -0800
Subject: [PATCH 18/27] [tests] create services for tests using the alpine
 image.

It is smaller, thus it should take less time to push and remove from registries.
Add a new line after each "echo" in tests.
---
 tests/spec_gantry_test_helper.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 14f3440..9082fb1 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -71,6 +71,7 @@ export SCHEDULE_NEXT_UPDATE_AT="Schedule next update at"
 export SLEEP_SECONDS_BEFORE_NEXT_UPDATE="Sleep [0-9]+ seconds before next update"
 
 export TEST_IMAGE_REMOVER="ghcr.io/shizunge/gantry-development"
+export TEST_SERVICE_IMAGE="alpine:latest"
 
 test_log() {
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^NONE$"  && return 0;
@@ -569,9 +570,9 @@ build_test_image() {
   fi
   local FILE=
   FILE=$(mktemp)
-  echo "FROM alpinelinux/docker-cli:latest" > "${FILE}"
+  echo "FROM ${TEST_SERVICE_IMAGE}" > "${FILE}"
   echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
-  pull_image_if_not_exist "alpinelinux/docker-cli:latest"
+  pull_image_if_not_exist "${TEST_SERVICE_IMAGE}"
   echo "Building ${IMAGE_WITH_TAG} from ${FILE}"
   timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
   rm "${FILE}"
@@ -582,7 +583,7 @@ build_and_push_test_image() {
   local TASK_SECONDS="${2}"
   local EXIT_SECONDS="${3}"
   build_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" "${EXIT_SECONDS}"
-  echo -n "Pushing image "
+  echo "Pushing image "
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
   docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") push --quiet "${IMAGE_WITH_TAG}"
@@ -590,7 +591,7 @@ build_and_push_test_image() {
 
 prune_local_test_image() {
   local IMAGE_WITH_TAG="${1}"
-  echo -n "Removing image ${IMAGE_WITH_TAG} "
+  echo "Removing image ${IMAGE_WITH_TAG} "
   docker image rm "${IMAGE_WITH_TAG}" --force &
 }
 
@@ -813,7 +814,7 @@ _start_replicated_job() {
 
 stop_service() {
   local SERVICE_NAME="${1}"
-  echo -n "Removing service "
+  echo "Removing service "
   docker service rm "${SERVICE_NAME}" &
 }
 

From 1ed4bc1f0d67e8b059e081df7d8868e7afdf0e93 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Thu, 21 Nov 2024 23:44:09 -0800
Subject: [PATCH 19/27] [gantry] follow the image remover's log and eval_cmd
 stdout.

In tests, check DONE_REMOVING_IMAGES to ensure all logs are printed.
---
 src/lib-common.sh                             | 345 +++++++++++-------
 src/lib-gantry.sh                             |   5 +-
 tests/gantry_cleanup_images_spec.sh           |   4 +
 tests/gantry_common_options_spec.sh           |  15 +-
 tests/gantry_filters_spec.sh                  |   4 +
 tests/gantry_jobs_spec.sh                     |  36 +-
 tests/gantry_login_docker_config_spec.sh      |   3 +
 tests/gantry_login_negative_spec.sh           |   8 +
 tests/gantry_login_spec.sh                    |   4 +
 tests/gantry_manifest_spec.sh                 |   6 +
 tests/gantry_notify_spec.sh                   |   6 +
 tests/gantry_parallel_spec.sh                 |   4 +
 tests/gantry_rollback_spec.sh                 |   5 +
 tests/gantry_service_no_running_tasks_spec.sh |   2 +
 tests/gantry_service_single_spec.sh           |   4 +
 tests/gantry_update_options_spec.sh           |   4 +
 tests/spec_gantry_test_helper.sh              |  20 +-
 17 files changed, 310 insertions(+), 165 deletions(-)

diff --git a/src/lib-common.sh b/src/lib-common.sh
index d3b3753..84a97a1 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -15,6 +15,20 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+_random_string() {
+  head /dev/urandom | LANG=C tr -dc 'a-zA-Z0-9' | head -c 8
+}
+
+_pipe_name() {
+  local BASE_NAME="${1:-pipe-base-name}"
+  local RANDOM_STR=
+  RANDOM_STR=$(_random_string)
+  local TIMESTAMP=
+  TIMESTAMP=$(date +%s)
+  local PIPE_NAME="/tmp/${BASE_NAME}-$$-${TIMESTAMP}-${RANDOM_STR}"
+  echo "${PIPE_NAME}"
+}
+
 # Run "grep -q" and avoid broken pipe errors.
 grep_q() {
   # "grep -q" will exit immediately when the first line of data matches, and leading to broken pipe errors.
@@ -173,16 +187,15 @@ _log_docker_node() {
 # 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg>
 _log_docker_line() {
   local LEVEL="INFO";
-  local TIME_DOCKER TIME TASK_NODE SCOPE NODE MESSAGE SPACE FIRST_WORD
+  local TIME_DOCKER TIME TASK_NODE SCOPE NODE MESSAGE FIRST_WORD
   TIME_DOCKER=$(extract_string "${*}" ' ' 1)
   TIME=$(_log_docker_time "${TIME_DOCKER}")
   TASK_NODE=$(extract_string "${*}" ' ' 2)
   SCOPE=$(_log_docker_scope "${TASK_NODE}");
   NODE=$(_log_docker_node "${TASK_NODE}");
   MESSAGE=$(extract_string "${*}" '|' 2-);
-  # Remove the leading space.
-  SPACE=$(extract_string "${MESSAGE}" ' ' 1)
-  [ -z "${SPACE}" ] && MESSAGE=$(extract_string "${MESSAGE}" ' ' 2-)
+  # Remove a single leading space.
+  [ "${MESSAGE:0:1}" = " " ] && MESSAGE="${MESSAGE:1}"
   FIRST_WORD=$(extract_string "${MESSAGE}" ' ' 1);
   if _log_level "${FIRST_WORD}" >/dev/null; then
     LEVEL=${FIRST_WORD};
@@ -191,6 +204,13 @@ _log_docker_line() {
   _log_formatter "${LEVEL}" "${TIME}" "${NODE}" "${SCOPE}" "${MESSAGE}";
 }
 
+_log_docker_multiple_lines() {
+  local LINE=
+  while read -r LINE; do
+    _log_docker_line "${LINE}"
+  done
+}
+
 # Usage: echo "${LOGS}" | log_lines INFO
 log_lines() {
   local LEVEL="${1}";
@@ -296,6 +316,24 @@ attach_tag_to_log_scope() {
   echo "${OLD_LOG_SCOPE}${SEP}${TAG}"
 }
 
+_eval_cmd_core() {
+  local STDOUT_CMD="${1}"; shift;
+  local CMD="${*}"
+  local PIPE_NAME=
+  PIPE_NAME="$(_pipe_name "eval-cmd-stdout-pipe")"
+  mkfifo "${PIPE_NAME}"
+  local PID=
+  eval "${STDOUT_CMD}" < "${PIPE_NAME}" &
+  PID="${!}"
+  local RETURN_VALUE=
+  # No redirect for stderr, unless it is done by the CMD.
+  eval "${CMD}" > "${PIPE_NAME}"
+  RETURN_VALUE=$?
+  wait "${PID}"
+  rm "${PIPE_NAME}"
+  return "${RETURN_VALUE}"
+}
+
 eval_cmd() {
   local TAG="${1}"; shift;
   local CMD="${*}"
@@ -304,21 +342,17 @@ eval_cmd() {
   LOG_SCOPE=$(attach_tag_to_log_scope "${TAG}")
   export LOG_SCOPE
   log INFO "Run ${TAG} command: ${CMD}"
-  local EVAL_OUTPUT=
-  local EVAL_RETURN=
-  EVAL_OUTPUT=$(mktemp)
-  eval "${CMD}" > "${EVAL_OUTPUT}"
-  EVAL_RETURN=$?
-  if [ "${EVAL_RETURN}" = "0" ]; then
-    log_lines INFO < "${EVAL_OUTPUT}"
+  local LOG_CMD="log_lines INFO"
+  local RETURN_VALUE=0
+  _eval_cmd_core "${LOG_CMD}" "${CMD}"
+  RETURN_VALUE=$?
+  if [ "${RETURN_VALUE}" = "0" ]; then
+    log INFO "Finish ${TAG} command."
   else
-    log_lines WARN < "${EVAL_OUTPUT}"
-    log WARN "${TAG} command returned a non-zero value ${EVAL_RETURN}."
+    log ERROR "Finish ${TAG} command with a non-zero return value ${RETURN_VALUE}."
   fi
-  rm "${EVAL_OUTPUT}"
-  log INFO "Finish ${TAG} command."
   export LOG_SCOPE="${OLD_LOG_SCOPE}"
-  return "${EVAL_RETURN}"
+  return "${RETURN_VALUE}"
 }
 
 swarm_network_arguments() {
@@ -349,36 +383,57 @@ _get_docker_command_name_arg() {
 }
 
 _get_docker_command_detach() {
-  if echo "${@}" | grep_q "--detach"; then
+  if echo "${@}" | grep_q "--detach=false"; then
+    echo "false"
+  elif echo "${@}" | grep_q "--detach"; then
+    # assume we find --detach or --detach=true.
     echo "true"
-    return 0
+  else
+    echo "false"
   fi
-  echo "false"
+  return 0
 }
 
-docker_service_logs () {
+docker_service_logs() {
   local SERVICE_NAME="${1}"
+  local LOG_CMD="_log_docker_multiple_lines"
+  local CMD="docker service logs --timestamps --no-task-ids ${SERVICE_NAME} 2>&1"
   local RETURN_VALUE=0
-  local LOGS=
-  if ! LOGS=$(docker service logs --timestamps --no-task-ids "${SERVICE_NAME}" 2>&1); then
-    log ERROR "Failed to obtain logs of service ${SERVICE_NAME}."
-    RETURN_VALUE=1
-  fi
-  echo "${LOGS}" |
-  while read -r LINE; do
-    _log_docker_line "${LINE}"
-  done
+  _eval_cmd_core "${LOG_CMD}" "${CMD}"
+  RETURN_VALUE=$?
+  [ "${RETURN_VALUE}" != 0 ] && log ERROR "Failed to obtain logs of service ${SERVICE_NAME}. Return code ${RETURN_VALUE}."
   return "${RETURN_VALUE}"
 }
 
-docker_service_logs_follow() {
+_docker_service_exists() {
   local SERVICE_NAME="${1}"
-  docker service logs --timestamps --no-task-ids --follow "${SERVICE_NAME}" 2>&1 |
-  while read -r LINE; do
-    _log_docker_line "${LINE}"
+  docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" >/dev/null 2>&1
+}
+
+_docker_wait_until_service_removed() {
+  local SERVICE_NAME="${1}"
+  while _docker_service_exists "${SERVICE_NAME}"; do
+    sleep 1s
   done
 }
 
+# "docker service logs --follow" does not stop when the service stops.
+# This function will check the status of the service and stop the "docker service logs" command.
+_docker_service_logs_follow_and_stop() {
+  local SERVICE_NAME="${1}"
+  ! _docker_service_exists "${SERVICE_NAME}" && return 1;
+  local PID=
+  docker service logs --timestamps --no-task-ids --follow "${SERVICE_NAME}" 2>&1 &
+  PID="${!}"
+  _docker_wait_until_service_removed "${SERVICE_NAME}"
+  kill "${PID}" 2>&1
+}
+
+docker_service_logs_follow() {
+  local SERVICE_NAME="${1}"
+  _docker_service_logs_follow_and_stop "${SERVICE_NAME}" | _log_docker_multiple_lines
+}
+
 _docker_service_task_states() {
   local SERVICE_NAME="${1}"
   # We won't get the return value of the command via $? if we use "local STATES=$(command)".
@@ -388,6 +443,7 @@ _docker_service_task_states() {
     return 1
   fi
   local NAME_LIST=
+  local LINE=
   echo "${STATES}" | while read -r LINE; do
     local NAME=
     local NODE_STATE_AND_ERROR=
@@ -401,71 +457,144 @@ _docker_service_task_states() {
   done
 }
 
-# Usage: wait_service_state <SERVICE_NAME> [--running] [--complete]
-# Wait for the service, usually a global job or a replicated job, to reach either running or complete state.
-# The function returns immediately when any of the tasks of the service fails.
+# Echo the return value from the tasks.
+# Return 0: All tasks reach the want state, or there is an error.
+# Return 1: Keep waiting.
+_all_tasks_reach_state() {
+  local WANT_STATE="${1}"
+  local CHECK_FAILURES="${2}"
+  local STATES="${3}"
+  local NUM_LINES=0
+  local NUM_STATES=0
+  local NUM_FAILS=0
+  local LINE=
+  while read -r LINE; do
+    [ -z "${LINE}" ] && continue;
+    NUM_LINES=$((NUM_LINES+1));
+    echo "${LINE}" | grep_q "${WANT_STATE}" && NUM_STATES=$((NUM_STATES+1));
+    "${CHECK_FAILURES}" && echo "${LINE}" | grep_q "Failed" && NUM_FAILS=$((NUM_FAILS+1));
+  done < <(echo "${STATES}")
+  if [ "${NUM_LINES}" -le 0 ]; then
+    # continue
+    return 1
+  fi
+  if [ "${NUM_STATES}" = "${NUM_LINES}" ]; then
+    # break
+    echo "0"
+    return 0
+  fi
+  if [ "${NUM_FAILS}" = 0 ]; then
+    # continue
+    return 1
+  fi
+  # Get return value of the task from the string "task: non-zero exit (1)".
+  local TASK_RETURN_VALUE=
+  TASK_RETURN_VALUE=$(echo "${STATES}" | grep "Failed" | sed -n 's/.*task: non-zero exit (\([0-9]\+\)).*/\1/p')
+  # Get the first error code.
+  local RETURN_VALUE=
+  RETURN_VALUE=$(extract_string "${TASK_RETURN_VALUE:-1}" ' ' 1)
+  # break
+  echo "${RETURN_VALUE}"
+  return 0
+}
+
+# Usage: wait_service_state <SERVICE_NAME> <WANT_STATE>
+# Wait for the service, usually a global job or a replicated job,
+# to reach either running or complete state.
+# Valid WANT_STATE includes "Running" and "Complete"
+# When the WANT_STATE is complete, the function returns immediately
+# when any of the tasks of the service fails.
 # In case of task failing, the function returns a non-zero value.
 wait_service_state() {
-  local SERVICE_NAME="${1}"; shift;
-  local WAIT_RUNNING WAIT_COMPLETE;
-  WAIT_RUNNING=$(echo "${@}" | grep_q "--running" && echo "true" || echo "false")
-  WAIT_COMPLETE=$(echo "${@}" | grep_q "--complete" && echo "true" || echo "false")
-  local RETURN_VALUE=0
-  local DOCKER_CMD_ERROR=1
+  local SERVICE_NAME="${1}";
+  local WANT_STATE="${2}";
+  local CHECK_FAILURES=false
+  [ "${WANT_STATE}" = "Complete" ] && CHECK_FAILURES=true
   local SLEEP_SECONDS=1
+  local DOCKER_CMD_ERROR=1
+  local RETURN_VALUE=0
   local STATES=
   while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do
-    if ! ("${WAIT_RUNNING}" || "${WAIT_COMPLETE}"); then
-      RETURN_VALUE=0
-      DOCKER_CMD_ERROR=0
-      break
-    fi
-    local NUM_LINES=0
-    local NUM_RUNS=0
-    local NUM_DONES=0
-    local NUM_FAILS=0
-    local LINE=
-    while read -r LINE; do
-      [ -z "${LINE}" ] && continue;
-      NUM_LINES=$((NUM_LINES+1));
-      echo "${LINE}" | grep_q "Running" && NUM_RUNS=$((NUM_RUNS+1));
-      echo "${LINE}" | grep_q "Complete" && NUM_DONES=$((NUM_DONES+1));
-      echo "${LINE}" | grep_q "Failed" && NUM_FAILS=$((NUM_FAILS+1));
-    done < <(echo "${STATES}")
-    if [ "${NUM_LINES}" -gt 0 ]; then
-      if "${WAIT_RUNNING}" && [ "${NUM_RUNS}" -eq "${NUM_LINES}" ]; then
-        RETURN_VALUE=0
-        DOCKER_CMD_ERROR=0
-        break
-      fi
-      if "${WAIT_COMPLETE}" && [ "${NUM_DONES}" -eq "${NUM_LINES}" ]; then
-        RETURN_VALUE=0
-        DOCKER_CMD_ERROR=0
-        break
-      fi
-      if "${WAIT_COMPLETE}" && [ "${NUM_FAILS}" -gt 0 ]; then
-        # Get return value of the task from the string "task: non-zero exit (1)".
-        local TASK_RETURN_VALUE=
-        TASK_RETURN_VALUE=$(echo "${STATES}" | grep "Failed" | sed -n 's/.*task: non-zero exit (\([0-9]\+\)).*/\1/p')
-        # Get the first error code.
-        RETURN_VALUE=$(extract_string "${TASK_RETURN_VALUE:-1}" ' ' 1)
-        DOCKER_CMD_ERROR=0
-        break
-      fi
-    fi
+    DOCKER_CMD_ERROR=0
+    RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break
     sleep "${SLEEP_SECONDS}"
+    DOCKER_CMD_ERROR=1
   done
   if [ "${DOCKER_CMD_ERROR}" != "0" ]; then
     log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${STATES}"
     return 1
   fi
   local LINE=
-  while read -r LINE; do
+  echo "${STATES}" | while read -r LINE; do
     log INFO "Service ${SERVICE_NAME}: ${LINE}."
-  done < <(echo "${STATES}")
+  done
   return "${RETURN_VALUE}"
 }
 
+docker_service_remove() {
+  local SERVICE_NAME="${1}"
+  local POST_COMMAND="${2}"
+  ! _docker_service_exists "${SERVICE_NAME}" && return 0
+  log INFO "Removing service ${SERVICE_NAME}."
+  local LOG=
+  if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then
+    log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}"
+    return 1
+  fi
+  if [ -n "${POST_COMMAND}" ]; then
+    eval "${POST_COMMAND}"
+  fi
+  log INFO "Removed service ${SERVICE_NAME}."
+  return 0
+}
+
+# Works with the service started (e.g. via docker_global_job) with --detach.
+docker_service_follow_logs_wait_complete() {
+  local SERVICE_NAME="${1}"
+  local PID=
+  docker_service_logs_follow "${SERVICE_NAME}" &
+  PID="${!}"
+  wait_service_state "${SERVICE_NAME}" "Complete"
+  docker_service_remove "${SERVICE_NAME}" "wait ${PID}"
+}
+
+# We do not expect failures when using docker_global_job.
+# Docker will try to restart the failed tasks.
+# We do not check the converge of the service, thus some jobs may failed on some nodes.
+# It is better to be used togther with wait_service_state.
+docker_global_job() {
+  local SERVICE_NAME=
+  SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
+  log INFO "Starting global-job ${SERVICE_NAME}."
+  local LOG=
+  if ! LOG=$(docker service create --mode global-job "${@}"  2>&1); then
+    log ERROR "Failed to create global-job ${SERVICE_NAME}: ${LOG}"
+    return 1
+  fi
+  return 0
+}
+
+# A job could fail when using docker_replicated_job.
+docker_replicated_job() {
+  local SERVICE_NAME=
+  local IS_DETACH=
+  SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
+  IS_DETACH=$(_get_docker_command_detach "${@}")
+  # Add "--detach" to work around https://github.com/docker/cli/issues/2979
+  # The Docker CLI does not exit on failures.
+  log INFO "Starting replicated-job ${SERVICE_NAME}."
+  local LOG=
+  if ! LOG=$(docker service create --mode replicated-job --detach "${@}" 2>&1); then
+    log ERROR "Failed to create replicated-job ${SERVICE_NAME}: ${LOG}"
+    return 1
+  fi
+  # If the command line does not contain '--detach', the function returns til the replicated job is complete.
+  if ! "${IS_DETACH}"; then
+    wait_service_state "${SERVICE_NAME}" "Complete" || return $?
+  fi
+  return 0
+}
+
 docker_version() {
   local cver capi sver sapi
   if ! cver=$(docker version --format '{{.Client.Version}}' 2>&1);    then log ERROR "${cver}"; cver="error"; fi
@@ -517,58 +646,6 @@ docker_current_container_name() {
   done
 }
 
-docker_service_remove() {
-  local SERVICE_NAME="${1}"
-  if ! docker service inspect --format '{{.JobStatus}}' "${SERVICE_NAME}" >/dev/null 2>&1; then
-    return 0
-  fi
-  log INFO "Removing service ${SERVICE_NAME}."
-  local LOG=
-  if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then
-    log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}"
-    return 1
-  fi
-  log INFO "Removed service ${SERVICE_NAME}."
-  return 0
-}
-
-# We do not expect failures when using docker_global_job.
-# Docker will try to restart the failed tasks.
-# We do not check the converge of the service, thus some jobs may failed on some nodes.
-# It is better to be used togther with wait_service_state.
-docker_global_job() {
-  local SERVICE_NAME=
-  SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
-  log INFO "Starting global-job ${SERVICE_NAME}."
-  local LOG=
-  if ! LOG=$(docker service create --mode global-job "${@}"  2>&1); then
-    log ERROR "Failed to create global-job ${SERVICE_NAME}: ${LOG}"
-    return 1
-  fi
-  return 0
-}
-
-# A job could fail when using docker_replicated_job.
-docker_replicated_job() {
-  local SERVICE_NAME=
-  local IS_DETACH=
-  SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
-  IS_DETACH=$(_get_docker_command_detach "${@}")
-  # Add "--detach" to work around https://github.com/docker/cli/issues/2979
-  # The Docker CLI does not exit on failures.
-  log INFO "Starting replicated-job ${SERVICE_NAME}."
-  local LOG=
-  if ! LOG=$(docker service create --mode replicated-job --detach "${@}" 2>&1); then
-    log ERROR "Failed to create replicated-job ${SERVICE_NAME}: ${LOG}"
-    return 1
-  fi
-  # If the command line does not contain '--detach', the function returns til the replicated job is complete.
-  if ! "${IS_DETACH}"; then
-    wait_service_state "${SERVICE_NAME}" --complete || return $?
-  fi
-  return 0
-}
-
 _container_status() {
   local CNAME="${1}"
   docker container inspect --format '{{.State.Status}}' "${CNAME}" 2>/dev/null
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index cd7cf60..c80117f 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -449,15 +449,14 @@ _remove_images() {
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
   docker_global_job --name "${SERVICE_NAME}" \
+    --detach=true \
     --restart-condition on-failure \
     --restart-max-attempts 1 \
     --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock \
     --env "GANTRY_IMAGES_TO_REMOVE=${IMAGES_TO_REMOVE_LIST}" \
     ${CLEANUP_IMAGES_OPTIONS} \
     "${IMAGES_REMOVER}";
-  wait_service_state "${SERVICE_NAME}"
-  docker_service_logs "${SERVICE_NAME}"
-  docker_service_remove "${SERVICE_NAME}"
+  docker_service_follow_logs_wait_complete "${SERVICE_NAME}"
 }
 
 _report_list() {
diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index 245b854..3189ed0 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -60,6 +60,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true" "coverage:true"
@@ -106,6 +107,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "Failed.*--incorrect-option"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true" "coverage:true"
@@ -151,6 +153,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "Failed.*--container-label=test"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" "coverage:true"
@@ -221,6 +224,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG0}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG1}"
       The stderr should satisfy spec_expect_message    "There is no image.*${IMAGE_WITH_TAG2}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Single service'
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index b6d0985..f659d7d 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -64,6 +64,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
@@ -146,8 +147,10 @@ Describe 'common-options'
       export GANTRY_CLEANUP_IMAGES=
       # Test that pre-run command can change the global configurations.
       export GANTRY_PRE_RUN_CMD="echo \"Pre update\"; GANTRY_UPDATE_OPTIONS=--detach=true; GANTRY_CLEANUP_IMAGES=false;"
+      # This command outputs multiple lines.
+      local POST_CMD="for I in \$(seq 3 5); do echo \"OUTPUT_LINE=\$I\"; done"
       # Test that the command returns a non-zero value.
-      export GANTRY_POST_RUN_CMD="echo \"Post update\"; false;"
+      export GANTRY_POST_RUN_CMD="echo \"Post update\"; ${POST_CMD}; false;"
       run_gantry "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -160,7 +163,7 @@ Describe 'common-options'
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "Pre update"
-      The stderr should satisfy spec_expect_no_message "pre-run command returned a non-zero value"
+      The stderr should satisfy spec_expect_message    "Finish pre-run command."
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -182,8 +185,12 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "Post update"
-      The stderr should satisfy spec_expect_message    "post-run command returned a non-zero value"
+      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=3"
+      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=4"
+      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=5"
+      The stderr should satisfy spec_expect_message    "Finish post-run command with a non-zero return value 1."
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
@@ -236,6 +243,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message        "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message        "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message        "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message        "${DONE_REMOVING_IMAGES}"
       # Check messages between iterations.
       The stderr should satisfy spec_expect_message           "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_message           "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
@@ -283,6 +291,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index 98be545..d14ebcb 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -61,6 +61,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true" "coverage:true"
@@ -118,6 +119,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_SERVICES_EXCLUDED_FILTERS_default" "container_test:true" "coverage:true"
@@ -168,6 +170,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
@@ -212,6 +215,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Filters'
\ No newline at end of file
diff --git a/tests/gantry_jobs_spec.sh b/tests/gantry_jobs_spec.sh
index 6bd7171..0bd26ae 100644
--- a/tests/gantry_jobs_spec.sh
+++ b/tests/gantry_jobs_spec.sh
@@ -19,12 +19,12 @@ Describe 'update-jobs'
   SUITE_NAME="update-jobs"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_update_jobs_skipping" "container_test:true" "coverage:true"
+  Describe "test_jobs_skipping" "container_test:true" "coverage:true"
     # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
     # This test also checks whether we do an extra step to to perform the exact match.
-    TEST_NAME="test_update_jobs_skipping"
+    TEST_NAME="test_jobs_skipping"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
@@ -36,7 +36,7 @@ Describe 'update-jobs'
       common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}"
       start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
     }
-    test_update_jobs_skipping() {
+    test_jobs_skipping() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -53,7 +53,7 @@ Describe 'update-jobs'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -83,16 +83,17 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_UPDATE_JOBS_true"
+  Describe "test_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+    TEST_NAME="test_jobs_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TASK_SECONDS=-1
     # Use a long EXIT_SECONDS, because we want the task keep running for a while after updating to trigger the image removing failure.
     EXIT_SECONDS=30
-    test_update_jobs_UPDATE_JOBS_true() {
+    test_jobs_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -104,7 +105,7 @@ Describe 'update-jobs'
     BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -133,16 +134,17 @@ Describe 'update-jobs'
       # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_label_UPDATE_JOBS_true"
+  Describe "test_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+    TEST_NAME="test_jobs_label_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TASK_SECONDS=-1
     # Use a long EXIT_SECONDS, because we want the task keep running for a while after updating to trigger the image removing failure.
     EXIT_SECONDS=30
-    test_update_jobs_label_UPDATE_JOBS_true() {
+    test_jobs_label_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -160,7 +162,7 @@ Describe 'update-jobs'
     BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -189,10 +191,11 @@ Describe 'update-jobs'
       # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_jobs_no_running_tasks" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_no_running_tasks"
+  Describe "test_jobs_no_running_tasks" "container_test:true" "coverage:true"
+    TEST_NAME="test_jobs_no_running_tasks"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     # Use a short TASK_SECONDS, because we want the task finishes soon.
@@ -206,7 +209,7 @@ Describe 'update-jobs'
       # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks.
       wait_zero_running_tasks "${SERVICE_NAME}"
     }
-    test_update_jobs_no_running_tasks() {
+    test_jobs_no_running_tasks() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -217,7 +220,7 @@ Describe 'update-jobs'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -247,6 +250,7 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'update-jobs'
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index 6afa5cb..d8a14db 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -103,6 +103,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_docker_config_default_config" "container_test:true" "coverage:true"
@@ -173,6 +174,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
@@ -291,6 +293,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'login_docker_config'
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index a54e61a..6db85d2 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -67,6 +67,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_incorrect_password" "container_test:false" "coverage:true"
@@ -135,6 +136,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_read_only_file" "container_test:false" "coverage:true"
@@ -205,6 +207,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_config_mismatch_default" "container_test:false" "coverage:true"
@@ -289,6 +292,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_config_mismatch_no_default" "container_test:false" "coverage:true"
@@ -367,6 +371,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_multi_services_no_label" "container_test:false" "coverage:true"
@@ -481,6 +486,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" "coverage:true"
@@ -550,6 +556,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_file_not_exist" "container_test:false" "coverage:true"
@@ -615,6 +622,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'login_negative'
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 1b5c0b4..4c26906 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -93,6 +93,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_default_config" "container_test:true" "coverage:true"
@@ -162,6 +163,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_REGISTRY_CONFIGS_FILE" "container_test:true" "coverage:true"
@@ -236,6 +238,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_external_config" "container_test:true" "coverage:true"
@@ -307,6 +310,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Login'
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index 493b21e..d79ca32 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -67,6 +67,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_MANIFEST_CMD_none_SERVICES_SELF" "container_test:true" "coverage:true"
@@ -114,6 +115,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_MANIFEST_CMD_manifest" "container_test:true" "coverage:true"
@@ -158,6 +160,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_MANIFEST_CMD_label" "container_test:true" "coverage:true"
@@ -207,6 +210,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false" "coverage:true"
@@ -253,6 +257,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_MANIFEST_CMD_failure" "container_test:false" "coverage:true"
@@ -307,6 +312,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Manifest command'
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index d20ad15..02dacdd 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -115,6 +115,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_NOTIFY_APPRISE}"
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
@@ -166,6 +167,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_NOTIFY_APPRISE}"
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
@@ -210,6 +212,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_NOTIFY_APPRISE}"
       The stderr should satisfy spec_expect_message    "Failed to send notification via Apprise"
     End
@@ -261,6 +264,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_NOTIFY_APPRISE}"
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
@@ -311,6 +315,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NO_UPDATES_OR_ERRORS_FOR_NOTIFICATION}"
       The stderr should satisfy spec_expect_message    "${SKIP_SENDING_NOTIFICATION}"
     End
@@ -362,6 +367,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_NOTIFY_APPRISE}"
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index 4d4e467..b1f77ab 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -82,6 +82,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_parallel_more_workers" "container_test:true" "coverage:true"
@@ -128,6 +129,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
@@ -171,6 +173,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
@@ -214,6 +217,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Multiple services'
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 256a355..0e561bf 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -65,6 +65,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_rollback_failed" "container_test:false" "coverage:true"
@@ -116,6 +117,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_rollback_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
@@ -165,6 +167,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_rollback_label_failed" "container_test:false" "coverage:true"
@@ -219,6 +222,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_rollback_label_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
@@ -270,6 +274,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Rollback'
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index 78ebc1e..f39be20 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -96,6 +96,7 @@ Describe "service-no-running-tasks"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_no_running_tasks_global" "container_test:true" "coverage:true"
@@ -156,6 +157,7 @@ Describe "service-no-running-tasks"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe "No Running Tasks"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index c37fedf..9daeb2b 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -61,6 +61,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
@@ -106,6 +107,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
@@ -162,6 +164,7 @@ Describe 'service-single-service'
       # Failed to removing the old image due to it has no digest.
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
@@ -211,6 +214,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 91bb74a..8d97030 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -78,6 +78,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_update_label_UPDATE_OPTIONS" "container_test:true" "coverage:true"
@@ -136,6 +137,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true"
@@ -180,6 +182,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_update_lable_UPDATE_TIMEOUT_SECONDS" "container_test:true" "coverage:true"
@@ -242,6 +245,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'update-options'
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 9082fb1..04787fd 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -67,6 +67,7 @@ export REMOVING_NUM_IMAGES="Removing [0-9]+ image\(s\)"
 export SKIP_REMOVING_IMAGES="Skip removing images"
 export REMOVED_IMAGE="Removed image"
 export FAILED_TO_REMOVE_IMAGE="Failed to remove image"
+export DONE_REMOVING_IMAGES="Done removing images"
 export SCHEDULE_NEXT_UPDATE_AT="Schedule next update at"
 export SLEEP_SECONDS_BEFORE_NEXT_UPDATE="Sleep [0-9]+ seconds before next update"
 
@@ -246,7 +247,8 @@ _login_test_registry() {
   local USERNAME="${3}"
   local PASSWORD="${4}"
   if ! _enforce_login_enabled "${ENFORCE_LOGIN}"; then
-    return 0
+    USERNAME="username"
+    PASSWORD="password"
   fi
   echo "Logging in ${REGISTRY}."
   local CONFIG=
@@ -522,7 +524,7 @@ _expect_message() {
   MESSAGE="${2}"
   local GREEN='\033[0;32m'
   local NO_COLOR='\033[0m'
-  if ! ACTUAL_MSG=$(echo "${TEXT}" | grep -Po "${MESSAGE}"); then
+  if ! ACTUAL_MSG=$(echo -e "${TEXT}" | grep -Po "${MESSAGE}"); then
     _handle_failure "Failed to find expected message \"${MESSAGE}\"."
     return 1
   fi
@@ -534,7 +536,7 @@ _expect_no_message() {
   MESSAGE="${2}"
   local GREEN='\033[0;32m'
   local NO_COLOR='\033[0m'
-  if ACTUAL_MSG=$(echo "${TEXT}" | grep -Po "${MESSAGE}"); then
+  if ACTUAL_MSG=$(echo -e "${TEXT}" | grep -Po "${MESSAGE}"); then
     _handle_failure "The following message should not present: \"${ACTUAL_MSG}\""
     return 1
   fi
@@ -573,7 +575,7 @@ build_test_image() {
   echo "FROM ${TEST_SERVICE_IMAGE}" > "${FILE}"
   echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
   pull_image_if_not_exist "${TEST_SERVICE_IMAGE}"
-  echo "Building ${IMAGE_WITH_TAG} from ${FILE}"
+  echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
   timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
   rm "${FILE}"
 }
@@ -583,7 +585,7 @@ build_and_push_test_image() {
   local TASK_SECONDS="${2}"
   local EXIT_SECONDS="${3}"
   build_test_image "${IMAGE_WITH_TAG}" "${TASK_SECONDS}" "${EXIT_SECONDS}"
-  echo "Pushing image "
+  echo "Pushing image ${IMAGE_WITH_TAG}"
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
   docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") push --quiet "${IMAGE_WITH_TAG}"
@@ -591,8 +593,8 @@ build_and_push_test_image() {
 
 prune_local_test_image() {
   local IMAGE_WITH_TAG="${1}"
-  echo "Removing image ${IMAGE_WITH_TAG} "
-  docker image rm "${IMAGE_WITH_TAG}" --force &
+  echo "Removing image ${IMAGE_WITH_TAG}"
+  docker image rm "${IMAGE_WITH_TAG}" --force
 }
 
 docker_service_update() {
@@ -814,8 +816,8 @@ _start_replicated_job() {
 
 stop_service() {
   local SERVICE_NAME="${1}"
-  echo "Removing service "
-  docker service rm "${SERVICE_NAME}" &
+  echo "Removing service ${SERVICE_NAME}"
+  docker service rm "${SERVICE_NAME}"
 }
 
 stop_multiple_services() {

From b0ff072b022dad7240d6564200950bf906bd2815 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Sat, 23 Nov 2024 18:21:09 -0800
Subject: [PATCH 20/27] [lib-common] increase docker log speed.

---
 src/lib-common.sh                | 253 +++++++++++++++++++------------
 src/lib-gantry.sh                |  17 +--
 tests/README.md                  |   2 +-
 tests/spec_gantry_test_helper.sh |   9 +-
 4 files changed, 169 insertions(+), 112 deletions(-)

diff --git a/src/lib-common.sh b/src/lib-common.sh
index 84a97a1..c01ee1f 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -29,6 +29,10 @@ _pipe_name() {
   echo "${PIPE_NAME}"
 }
 
+_get_first_word() {
+  echo "${*}" | sed -n -E "s/^(\S+).*/\1/p";
+}
+
 # Run "grep -q" and avoid broken pipe errors.
 grep_q() {
   # "grep -q" will exit immediately when the first line of data matches, and leading to broken pipe errors.
@@ -72,19 +76,79 @@ extract_string() {
   echo "${ECHO_STRING}" | cut -d "${DELIMITER}" -f "${POSITION}"
 }
 
-# echo the number of the log level.
-# return 0 if LEVEL is supported.
-# return 1 if LEVLE is unsupported.
-_log_level() {
+# All lower or all upper. No mix.
+_log_level_to_upper() {
+  local LEVEL="${1}";
+  case "${LEVEL}" in
+    "debug") echo "DEBUG"; ;;
+    "info")  echo "INFO";  ;;
+    "warn")  echo "WARN";  ;;
+    "error") echo "ERROR"; ;;
+    "none")  echo "NONE";  ;;
+    *) echo "${LEVEL}"; ;;
+  esac
+}
+
+# Return 0 if the first work is a supported level.
+# Return 1 else.
+_first_word_is_level() {
+  local MSG="${1}"
+  local LEN="${#MSG}"
+  local LEVEL=
+  [ "${LEN}" -lt 4 ] && return 1
+  if [ "${LEN}" = 4 ] || [ "${MSG:4:1}" = " " ]; then
+    LEVEL="${MSG:0:4}"
+  elif [ "${LEN}" = 5 ] || [ "${MSG:5:1}" = " " ]; then
+    LEVEL="${MSG:0:5}"
+  else
+    return 1
+  fi
+  LEVEL=$(_log_level_to_upper "${LEVEL}")
+  case "${LEVEL}" in
+    "DEBUG") return 0; ;;
+    "INFO")  return 0; ;;
+    "WARN")  return 0; ;;
+    "ERROR") return 0; ;;
+    "NONE")  return 0; ;;
+    *) return 1; ;;
+  esac
+}
+
+_log_skip_level() {
   local LEVEL="${1}";
-  [ -z "${LEVEL}" ] && _log_level "INFO" && return 1;
-  echo "${LEVEL}" | grep_q_i "^DEBUG$" && echo 0 && return 0;
-  echo "${LEVEL}" | grep_q_i "^INFO$"  && echo 1 && return 0;
-  echo "${LEVEL}" | grep_q_i "^WARN$"  && echo 2 && return 0;
-  echo "${LEVEL}" | grep_q_i "^ERROR$" && echo 3 && return 0;
-  echo "${LEVEL}" | grep_q_i "^NONE$"  && echo 4 && return 0;
-  _log_level "NONE";
-  return 1;
+  case "${LEVEL}" in
+    "DEBUG")   return "${2}"; ;;
+    "INFO"|"") return "${3}"; ;;
+    "WARN")    return "${4}"; ;;
+    "ERROR")   return "${5}"; ;;
+    "NONE"|*)  return "${6}"; ;;
+  esac
+}
+
+# return 0 to skip logging.
+# return 1 otherwise.
+_log_skip() {
+  local TARGET_LEVEL="${1}";
+  local LEVEL="${2}";
+  # This is 10% faster than the following command:
+  # _log_level() {
+  #   local LEVEL="${1}";
+  #   case "${LEVEL}" in
+  #     "DEBUG") echo 0; ;;
+  #     "INFO"|"") echo 1; ;;
+  #     "WARN") echo 2; ;;
+  #     "ERROR") echo 3; ;;
+  #     "NONE"|*) echo 4; ;;
+  #   esac
+  # }
+  # test "$(_log_level "${LEVEL}")" -lt "$(_log_level "${TARGET_LEVEL}")"; return $?
+  case "${TARGET_LEVEL}" in
+    "DEBUG")   _log_skip_level "${LEVEL}" 1 1 1 1 1; ;;
+    "INFO"|"") _log_skip_level "${LEVEL}" 0 1 1 1 1; ;;
+    "WARN")    _log_skip_level "${LEVEL}" 0 0 1 1 1; ;;
+    "ERROR")   _log_skip_level "${LEVEL}" 0 0 0 1 1; ;;
+    "NONE"|*)  _log_skip_level "${LEVEL}" 0 0 0 0 1; ;;
+  esac
 }
 
 _level_color() {
@@ -94,114 +158,109 @@ _level_color() {
   local ORANGE='\033[0;33m'
   local GREEN='\033[0;32m'
   local BLUE='\033[0;34m'
-  echo "${LEVEL}" | grep_q_i "^DEBUG$" && echo "${BLUE}" && return 0;
-  echo "${LEVEL}" | grep_q_i "^INFO$"  && echo "${GREEN}" && return 0;
-  echo "${LEVEL}" | grep_q_i "^WARN$"  && echo "${ORANGE}" && return 0;
-  echo "${LEVEL}" | grep_q_i "^ERROR$" && echo "${RED}" && return 0;
-  echo "${NO_COLOR}"
+  case "${LEVEL}" in
+    "DEBUG") echo "${BLUE}"; ;;
+    "INFO")  echo "${GREEN}"; ;;
+    "WARN")  echo "${ORANGE}"; ;;
+    "ERROR") echo "${RED}"; ;;
+    *)       echo "${NO_COLOR}"; ;;
+  esac
+  return 0;
 }
 
 _color_iso_time() {
-  # Highlight time within the day in ISO-8601
-  # \\033[1;30m : Dark Gray
-  # \\033[0;37m : Ligth Gray
-  # \\033[0m    : No color
-  echo "${*}" | sed -E 's/(.*[0-9]+-[0-9]+-[0-9]+)T([0-9]+:[0-9]+:[0-9]+)(.*)/\\033[1;30m\1T\\033[0;37m\2\\033[1;30m\3\\033[0m/'
+  local EPOCH="${1}"
+  # Highlight time within the day in ISO-8601 (2024-11-23T21:50:13-08:00)
+  # local NO_COLOR='\033[0m'
+  # local DGRAY="\033[1;30m"
+  # local LGRAY="\033[0;37m"
+  local TIME_STR=
+  TIME_STR=$(busybox date -d "@${EPOCH}" +"\033[1;30m%Y-%m-%dT\033[0;37m%H:%M:%S\033[1;30m%z")
+  # +0000 -> +00:00
+  echo "${TIME_STR:0:-2}:${TIME_STR:0-2}\033[0m"
 }
 
 _log_formatter() {
-  local LOG_LEVEL="${LOG_LEVEL}"
-  local LEVEL="${1}"; shift;
-  [ "$(_log_level "${LEVEL}")" -lt "$(_log_level "${LOG_LEVEL}")" ] && return 0;
-  LEVEL=$(echo "${LEVEL}" | tr '[:lower:]' '[:upper:]')
-  local TIME="${1}"; shift;
-  local LOCATION="${1}"; shift;
-  local SCOPE="${1}"; shift;
+  local TARGET_LEVEL="${LOG_LEVEL:-}";
+  local LEVEL="${1}";
+  local EPOCH="${2}";
+  local LOCATION="${3}";
+  local SCOPE="${4}";
+  TARGET_LEVEL=$(_log_level_to_upper "${TARGET_LEVEL}")
+  LEVEL=$(_log_level_to_upper "${LEVEL}")
+  _log_skip "${TARGET_LEVEL}" "${LEVEL}" && return 0;
+  shift 4;
   local NO_COLOR='\033[0m'
   local DGRAY='\033[1;30m'
-  local MSG=
-  MSG="${DGRAY}[$(_color_iso_time "${TIME}")${DGRAY}]${NO_COLOR}"
+  local TIME_STR LOC_STR LEVEL_STR SCOPE_STR
+  TIME_STR="${DGRAY}[$(_color_iso_time "${EPOCH}")${DGRAY}]${NO_COLOR}"
   if [ -n "${LOCATION}" ]; then
-    MSG="${MSG}${DGRAY}[${LOCATION}]${NO_COLOR}"
+    LOC_STR="${DGRAY}[${LOCATION}]${NO_COLOR}"
   fi
-  MSG="${MSG}$(_level_color "${LEVEL}")[${LEVEL}]${NO_COLOR} "
+  LEVEL_STR="$(_level_color "${LEVEL}")[${LEVEL}]${NO_COLOR} "
   if [ -n "${SCOPE}" ]; then
-    MSG="${MSG}${DGRAY}${SCOPE}:${NO_COLOR} "
+    SCOPE_STR="${DGRAY}${SCOPE}:${NO_COLOR} "
   fi
-  MSG="${MSG}$(echo "${*}" | tr '\n' ' ')"
-  echo -e "${MSG}" >&2
+  local MSG_STR=
+  MSG_STR=$(echo "${*}" | tr '\n' ' ')
+  echo -e "${TIME_STR}${LOC_STR}${LEVEL_STR}${SCOPE_STR}${MSG_STR}" >&2
 }
 
 # We want to print an empty line for log without an argument. Thus we do not run the following check.
 # [ -z "${1}" ] && return 0
 log() {
-  local NODE_NAME="${NODE_NAME}"
-  local LOG_SCOPE="${LOG_SCOPE}"
+  local LOCAL_NODE="${NODE_NAME:-}"
+  local LOCAL_SCOPE="${LOG_SCOPE:-}"
   local LEVEL="INFO";
-  if _log_level "${1}" >/dev/null; then
+  if _first_word_is_level "${1}"; then
     LEVEL="${1}";
     shift;
   fi
-  _log_formatter "${LEVEL}" "$(date -Iseconds)" "${NODE_NAME}" "${LOG_SCOPE}" "${@}";
+  local EPOCH=
+  EPOCH="$(date +%s)"
+  _log_formatter "${LEVEL}" "${EPOCH}" "${LOCAL_NODE}" "${LOCAL_SCOPE}" "${@}";
 }
 
-_log_docker_time() {
-  # Convert timestamps from `docker service logs` to ISO-8601. The timestamps is in UTC.
+_log_docker_time_epoch() {
+  # Convert timestamps from `docker service logs` to EPOCH.
+  # The timestamps is in UTC.
   # docker service logs --timestamps --no-task-ids <service>
-  # 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg>
+  # 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg with spaces>
   local TIME_INPUT="${1}"
-  local EPOCH=
-  if ! EPOCH="$(busybox date -d "${TIME_INPUT}" -D "%Y-%m-%dT%H:%M:%S" -u +%s 2>/dev/null)"; then
-    date -Iseconds
+  if ! busybox date -d "${TIME_INPUT}" -D "%Y-%m-%dT%H:%M:%S" -u +%s 2>/dev/null; then
+    date +%s
     return 1
   fi
-  busybox date -d "@${EPOCH}" -Iseconds 2>&1
-}
-
-_log_docker_scope() {
-  local LOG_SCOPE="${LOG_SCOPE}"
-  local TASK_NODE="${1}"
-  local SCOPE=
-  SCOPE=$(echo "${TASK_NODE}" | sed -n "s/\(.*\)@.*/\1/p");
-  if [ -z "${SCOPE}" ]; then
-    echo "${LOG_SCOPE}"
-    return 1
-  fi
-  echo "${SCOPE}"
-}
-
-_log_docker_node() {
-  local NODE_NAME="${NODE_NAME}"
-  local TASK_NODE="${1}"
-  local NODE=
-  NODE=$(echo "${TASK_NODE}" | sed -n "s/.*@\(.*\)/\1/p");
-  if [ -z "${NODE}" ]; then
-    echo "${NODE_NAME}"
-    return 1
-  fi
-  echo "${NODE}"
 }
 
 # Convert logs from `docker service logs` to `log` format.
 # docker service logs --timestamps --no-task-ids <service>
-# 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg>
+# 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg with spaces>
 _log_docker_line() {
+  local LOCAL_NODE="${NODE_NAME:-}"
+  local LOCAL_SCOPE="${LOG_SCOPE:-}"
   local LEVEL="INFO";
-  local TIME_DOCKER TIME TASK_NODE SCOPE NODE MESSAGE FIRST_WORD
-  TIME_DOCKER=$(extract_string "${*}" ' ' 1)
-  TIME=$(_log_docker_time "${TIME_DOCKER}")
-  TASK_NODE=$(extract_string "${*}" ' ' 2)
-  SCOPE=$(_log_docker_scope "${TASK_NODE}");
-  NODE=$(_log_docker_node "${TASK_NODE}");
-  MESSAGE=$(extract_string "${*}" '|' 2-);
-  # Remove a single leading space.
-  [ "${MESSAGE:0:1}" = " " ] && MESSAGE="${MESSAGE:1}"
-  FIRST_WORD=$(extract_string "${MESSAGE}" ' ' 1);
-  if _log_level "${FIRST_WORD}" >/dev/null; then
-    LEVEL=${FIRST_WORD};
-    MESSAGE=$(extract_string "${MESSAGE}" ' ' 2-);
+  local EPOCH NODE SCOPE MESSAGE
+  # Using the same regexp for all 4 parts:
+  local TIME_DOCKER TASK_DOCKER NODE_DOCKER
+  read -r TIME_DOCKER TASK_DOCKER NODE_DOCKER MESSAGE < <(echo "${*}" | sed -n -E "s/^(\S+) +(\S+)@(\S+) +\| ?/\1 \2 \3 /p");
+  EPOCH=$(_log_docker_time_epoch "${TIME_DOCKER}");
+  if [ -n "${TASK_DOCKER}" ] || [ -n "${NODE_DOCKER}" ] || [ -n "${MESSAGE}" ]; then
+    NODE="${NODE_DOCKER}"
+    SCOPE="${TASK_DOCKER}"
+    if _first_word_is_level "${MESSAGE}"; then
+      LEVEL=$(_get_first_word "${MESSAGE}");
+      MESSAGE=$(extract_string "${MESSAGE}" ' ' 2-);
+    fi
+  else
+    # All three are empty, sed failure indicates errors.
+    # format error, imply that we receive an error message from the "docker service logs" command.
+    LEVEL="ERROR"
+    NODE="${LOCAL_NODE}"
+    SCOPE="${LOCAL_SCOPE}"
+    MESSAGE="${*}"
   fi
-  _log_formatter "${LEVEL}" "${TIME}" "${NODE}" "${SCOPE}" "${MESSAGE}";
+  _log_formatter "${LEVEL}" "${EPOCH}" "${NODE}" "${SCOPE}" "${MESSAGE}";
 }
 
 _log_docker_multiple_lines() {
@@ -227,8 +286,7 @@ is_number() {
 
 is_true() {
   local CONFIG="${1}"
-  CONFIG=$(extract_string "${CONFIG}" ' ' 1)
-  echo "${CONFIG}" | grep_q_i "true"
+  echo "${CONFIG}" | grep_q_i "^true$"
 }
 
 first_minus_second() {
@@ -277,7 +335,7 @@ add_unique_to_list() {
   echo -e "${OLD_LIST}\n${NEW_ITEM}" | sort | uniq
 }
 
-# For a givne variable name <VAR>, try to read content of <VAR>_FILE if file exists.
+# For a given variable name <VAR>, try to read content of <VAR>_FILE if file exists.
 # otherwise echo the content of <VAR>.
 read_config() {
   local CONFIG_NAME="${1}"
@@ -338,7 +396,7 @@ eval_cmd() {
   local TAG="${1}"; shift;
   local CMD="${*}"
   [ -z "${CMD}" ] && return 0
-  local OLD_LOG_SCOPE="${LOG_SCOPE}"
+  local OLD_LOG_SCOPE="${LOG_SCOPE:-}"
   LOG_SCOPE=$(attach_tag_to_log_scope "${TAG}")
   export LOG_SCOPE
   log INFO "Run ${TAG} command: ${CMD}"
@@ -379,7 +437,7 @@ timezone_arguments() {
 
 _get_docker_command_name_arg() {
   # get <NAME> from "--name <NAME>" or "--name=<NAME>"
-  echo "${@}" | tr '\n' ' ' | sed -E 's/.*--name[ =]([^ ]*).*/\1/'
+  echo "${@}" | tr '\n' ' ' | sed -n -E 's/.*--name[ =](\S*).*/\1/p'
 }
 
 _get_docker_command_detach() {
@@ -426,7 +484,8 @@ _docker_service_logs_follow_and_stop() {
   docker service logs --timestamps --no-task-ids --follow "${SERVICE_NAME}" 2>&1 &
   PID="${!}"
   _docker_wait_until_service_removed "${SERVICE_NAME}"
-  kill "${PID}" 2>&1
+  # Use kill signal to avoid an additional "context canceled" message from the Term or Int signal.
+  kill -kill "${PID}" 2>&1
 }
 
 docker_service_logs_follow() {
@@ -489,10 +548,10 @@ _all_tasks_reach_state() {
   fi
   # Get return value of the task from the string "task: non-zero exit (1)".
   local TASK_RETURN_VALUE=
-  TASK_RETURN_VALUE=$(echo "${STATES}" | grep "Failed" | sed -n 's/.*task: non-zero exit (\([0-9]\+\)).*/\1/p')
+  TASK_RETURN_VALUE=$(echo "${STATES}" | grep "Failed" | sed -n -E 's/.*task: non-zero exit \(([0-9]+)\).*/\1/p')
   # Get the first error code.
   local RETURN_VALUE=
-  RETURN_VALUE=$(extract_string "${TASK_RETURN_VALUE:-1}" ' ' 1)
+  RETURN_VALUE=$(_get_first_word "${TASK_RETURN_VALUE:-1}")
   # break
   echo "${RETURN_VALUE}"
   return 0
@@ -561,7 +620,7 @@ docker_service_follow_logs_wait_complete() {
 # We do not expect failures when using docker_global_job.
 # Docker will try to restart the failed tasks.
 # We do not check the converge of the service, thus some jobs may failed on some nodes.
-# It is better to be used togther with wait_service_state.
+# It is better to be used together with wait_service_state.
 docker_global_job() {
   local SERVICE_NAME=
   SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
@@ -612,7 +671,9 @@ docker_current_container_name() {
   ALL_NETWORKS=$(docker network ls --format '{{.ID}}') || return 1;
   [ -z "${ALL_NETWORKS}" ] && return 0;
   local IPS=;
-  IPS=$(ip route | grep src | sed -n "s/.* src \(\S*\).*$/\1/p");
+  # Get the string after "src":
+  # 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown
+  IPS=$(ip route | grep src | sed -n -E "s/.* src (\S+).*$/\1/p");
   [ -z "${IPS}" ] && return 0;
   local GWBRIDGE_NETWORK HOST_NETWORK;
   GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1;
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index c80117f..18766a6 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -205,11 +205,7 @@ _authenticate_to_registries() {
     [ "${LINE:0:1}" = "#" ] && continue
     LINE=$(echo "${LINE}" | tr '\t' ' ')
     local OTHERS=
-    CONFIG=$(extract_string "${LINE}" ' ' 1)
-    HOST=$(extract_string "${LINE}" ' ' 2)
-    USER=$(extract_string "${LINE}" ' ' 3)
-    PASSWORD=$(extract_string "${LINE}" ' ' 4)
-    OTHERS=$(extract_string "${LINE}" ' ' 5-)
+    read -r CONFIG HOST USER PASSWORD OTHERS < <(echo "${LINE}")
     local ERROR_MSG=
     if [ -n "${OTHERS}" ]; then
       ERROR_MSG="Found extra item(s)."
@@ -460,8 +456,9 @@ _remove_images() {
 }
 
 _report_list() {
-  local PRE="${1}"; shift
-  local POST="${1}"; shift
+  local PRE="${1}";
+  local POST="${2}";
+  shift 2;
   local LIST="${*}"
   local NUM=
   NUM=$(_get_number_of_elements "${LIST}")
@@ -621,7 +618,7 @@ gantry_current_service_name() {
   local SNAME=
   SNAME=$(docker container inspect "${CNAME}" --format '{{range $key,$value := .Config.Labels}}{{$key}}={{println $value}}{{end}}' \
     | grep "com.docker.swarm.service.name" \
-    | sed -n "s/com.docker.swarm.service.name=\(.*\)$/\1/p") || return 1
+    | sed -n -E "s/com.docker.swarm.service.name=(.*)$/\1/p") || return 1
   _static_variable_add_unique_to_list STATIC_VAR_CURRENT_SERVICE_NAME "${SNAME}"
   echo "${SNAME}"
 }
@@ -666,7 +663,7 @@ _get_service_mode() {
   # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
   # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
   # We do an extra step to to perform the exact match.
-  MODE=$(echo "${MODE}" | sed -n "s/\(.*\) ${SERVICE_NAME}$/\1/p")
+  MODE=$(echo "${MODE}" | sed -n -E "s/(.*) ${SERVICE_NAME}$/\1/p")
   echo "${MODE}"
 }
 
@@ -880,7 +877,7 @@ _get_number_of_running_tasks() {
   # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
   # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
   # We do an extra step to to perform the exact match.
-  REPLICAS=$(echo "${REPLICAS}" | sed -n "s/\(.*\) ${SERVICE_NAME}$/\1/p")
+  REPLICAS=$(echo "${REPLICAS}" | sed -n -E "s/(.*) ${SERVICE_NAME}$/\1/p")
   # https://docs.docker.com/engine/reference/commandline/service_ls/#examples
   # The REPLICAS is like "5/5" or "1/1 (3/5 completed)"
   # Get the number before the first "/".
diff --git a/tests/README.md b/tests/README.md
index 1da24c1..b334374 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -29,7 +29,7 @@ bash shellspec --jobs 50
 
 To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) installed):
 ```
-bash shellspec --kcov
+bash shellspec --kcov --tag coverage:true
 ```
 
 If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 04787fd..0d4be29 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -259,13 +259,12 @@ _login_test_registry() {
 _logout_test_registry() {
   local ENFORCE_LOGIN="${1}"
   local REGISTRY="${2}"
-  if ! _enforce_login_enabled "${ENFORCE_LOGIN}"; then
-    return 0
-  fi
-  echo "Logging out ${REGISTRY}."
   local CONFIG=
   CONFIG=$(_get_docker_config_file "${REGISTRY}") || return 1
-  docker --config "${CONFIG}" logout
+  if _enforce_login_enabled "${ENFORCE_LOGIN}"; then
+    echo "Logging out ${REGISTRY}."
+    docker --config "${CONFIG}" logout
+  fi
   [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
 }
 

From a0d96e47f791b9e5f11d360095834f228098c0f2 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Sun, 24 Nov 2024 22:32:19 -0800
Subject: [PATCH 21/27] [tests] allow more tests to run with a container.

Use busybox time explicitly for docker service update.
---
 .github/workflows/coverage.yml                |   2 +-
 .github/workflows/on-push.yml                 |   2 +-
 src/docker_hub_rate.sh                        |   4 +-
 src/lib-common.sh                             | 208 ++++++++++--------
 src/lib-gantry.sh                             |  14 +-
 tests/README.md                               |   7 +-
 tests/gantry_cleanup_images_spec.sh           |   8 +-
 tests/gantry_common_options_spec.sh           |  61 +++--
 tests/gantry_filters_spec.sh                  |   8 +-
 tests/gantry_jobs_spec.sh                     |  22 +-
 tests/gantry_login_docker_config_spec.sh      |  51 +++--
 tests/gantry_login_negative_spec.sh           | 112 ++++++----
 tests/gantry_login_spec.sh                    |  40 ++--
 tests/gantry_manifest_spec.sh                 |  12 +-
 tests/gantry_notify_spec.sh                   |  12 +-
 tests/gantry_parallel_spec.sh                 |   8 +-
 tests/gantry_rollback_spec.sh                 |  30 +--
 tests/gantry_service_multiple_spec.sh         |   2 +-
 tests/gantry_service_no_running_tasks_spec.sh |   4 +-
 tests/gantry_service_single_spec.sh           |  20 +-
 tests/gantry_update_options_spec.sh           |  16 +-
 tests/spec_gantry_test_helper.sh              | 156 ++++++++-----
 22 files changed, 460 insertions(+), 339 deletions(-)

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 9d5e6c3..34d7321 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -38,7 +38,7 @@ jobs:
         run: |
           export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
           export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
-          bash shellspec --kcov --jobs 50 --tag "coverage:true"
+          bash shellspec --kcov --jobs 50
       - name: Upload reports
         uses: actions/upload-artifact@v4
         with:
diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml
index 127ceae..84ae731 100644
--- a/.github/workflows/on-push.yml
+++ b/.github/workflows/on-push.yml
@@ -162,4 +162,4 @@ jobs:
           export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
           export GANTRY_TEST_CONTAINER_REPO_TAG=$(cat tag.txt)
           echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
-          bash shellspec --jobs 50 --tag "container_test:true"
+          bash shellspec --jobs 50
diff --git a/src/docker_hub_rate.sh b/src/docker_hub_rate.sh
index 070d153..0d46fe7 100755
--- a/src/docker_hub_rate.sh
+++ b/src/docker_hub_rate.sh
@@ -79,7 +79,7 @@ docker_hub_rate() {
     return 1
   fi
   local TOKEN=
-  TOKEN=$(echo "${RESPONSE}" | sed 's/.*"token":"\([^"]*\).*/\1/')
+  TOKEN=$(echo "${RESPONSE}" | sed -E 's/.*"token":"([^"]*).*/\1/')
   if [ -z "${TOKEN}" ]; then
     _docker_hub_echo_error "PARSE TOKEN ERROR" "${RESPONSE}"
     return 1
@@ -94,7 +94,7 @@ docker_hub_rate() {
     return 1
   fi
   local RATE=
-  RATE=$(echo "${RESPONSE}" | sed -n 's/.*ratelimit-remaining: \([-]\?[0-9]\+\);.*/\1/p' )
+  RATE=$(echo "${RESPONSE}" | sed -n -E 's/.*ratelimit-remaining: (-?[0-9]+);.*/\1/p' )
   if [ -z "${RATE}" ]; then
     _docker_hub_echo_error "PARSE RATE ERROR" "${RESPONSE}"
     return 1
diff --git a/src/lib-common.sh b/src/lib-common.sh
index c01ee1f..c8d8bc5 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -21,11 +21,12 @@ _random_string() {
 
 _pipe_name() {
   local BASE_NAME="${1:-pipe-base-name}"
-  local RANDOM_STR=
-  RANDOM_STR=$(_random_string)
+  local PID=$$
   local TIMESTAMP=
   TIMESTAMP=$(date +%s)
-  local PIPE_NAME="/tmp/${BASE_NAME}-$$-${TIMESTAMP}-${RANDOM_STR}"
+  local RANDOM_STR=
+  RANDOM_STR=$(_random_string)
+  local PIPE_NAME="/tmp/${BASE_NAME}-${PID}-${TIMESTAMP}-${RANDOM_STR}"
   echo "${PIPE_NAME}"
 }
 
@@ -79,6 +80,7 @@ extract_string() {
 # All lower or all upper. No mix.
 _log_level_to_upper() {
   local LEVEL="${1}";
+  # tr is slow.
   case "${LEVEL}" in
     "debug") echo "DEBUG"; ;;
     "info")  echo "INFO";  ;;
@@ -114,20 +116,30 @@ _first_word_is_level() {
   esac
 }
 
-_log_skip_level() {
+_log_skip_level_echo_color() {
   local LEVEL="${1}";
+  # Ideally, one function should do one thing.
+  # But by merging two functions "_log_skip" and "log_color" into one, we reduce the number of "case" to improve performance.
+  # local BLUE='\033[0;34m'
+  # local GREEN='\033[0;32m'
+  # local ORANGE='\033[0;33m'
+  # local RED='\033[0;31m'
+  # local NO_COLOR='\033[0m'
+  # SC2028 (info): echo may not expand escape sequences. Use printf.
+  # shellcheck disable=SC2028
   case "${LEVEL}" in
-    "DEBUG")   return "${2}"; ;;
-    "INFO"|"") return "${3}"; ;;
-    "WARN")    return "${4}"; ;;
-    "ERROR")   return "${5}"; ;;
-    "NONE"|*)  return "${6}"; ;;
+    "DEBUG")   echo "\033[0;34m"; return "${2}"; ;;
+    "INFO"|"") echo "\033[0;32m"; return "${3}"; ;;
+    "WARN")    echo "\033[0;33m"; return "${4}"; ;;
+    "ERROR")   echo "\033[0;31m"; return "${5}"; ;;
+    "NONE"|*)  echo "\033[0m";    return "${6}"; ;;
   esac
 }
 
+# Echo the color for the given LEVEL.
 # return 0 to skip logging.
 # return 1 otherwise.
-_log_skip() {
+_log_skip_echo_color() {
   local TARGET_LEVEL="${1}";
   local LEVEL="${2}";
   # This is 10% faster than the following command:
@@ -143,67 +155,64 @@ _log_skip() {
   # }
   # test "$(_log_level "${LEVEL}")" -lt "$(_log_level "${TARGET_LEVEL}")"; return $?
   case "${TARGET_LEVEL}" in
-    "DEBUG")   _log_skip_level "${LEVEL}" 1 1 1 1 1; ;;
-    "INFO"|"") _log_skip_level "${LEVEL}" 0 1 1 1 1; ;;
-    "WARN")    _log_skip_level "${LEVEL}" 0 0 1 1 1; ;;
-    "ERROR")   _log_skip_level "${LEVEL}" 0 0 0 1 1; ;;
-    "NONE"|*)  _log_skip_level "${LEVEL}" 0 0 0 0 1; ;;
-  esac
-}
-
-_level_color() {
-  local LEVEL="${1}"
-  local NO_COLOR='\033[0m'
-  local RED='\033[0;31m'
-  local ORANGE='\033[0;33m'
-  local GREEN='\033[0;32m'
-  local BLUE='\033[0;34m'
-  case "${LEVEL}" in
-    "DEBUG") echo "${BLUE}"; ;;
-    "INFO")  echo "${GREEN}"; ;;
-    "WARN")  echo "${ORANGE}"; ;;
-    "ERROR") echo "${RED}"; ;;
-    *)       echo "${NO_COLOR}"; ;;
+    "DEBUG")   _log_skip_level_echo_color "${LEVEL}" 1 1 1 1 1; ;;
+    "INFO"|"") _log_skip_level_echo_color "${LEVEL}" 0 1 1 1 1; ;;
+    "WARN")    _log_skip_level_echo_color "${LEVEL}" 0 0 1 1 1; ;;
+    "ERROR")   _log_skip_level_echo_color "${LEVEL}" 0 0 0 1 1; ;;
+    "NONE"|*)  _log_skip_level_echo_color "${LEVEL}" 0 0 0 0 1; ;;
   esac
-  return 0;
-}
-
-_color_iso_time() {
-  local EPOCH="${1}"
-  # Highlight time within the day in ISO-8601 (2024-11-23T21:50:13-08:00)
-  # local NO_COLOR='\033[0m'
-  # local DGRAY="\033[1;30m"
-  # local LGRAY="\033[0;37m"
-  local TIME_STR=
-  TIME_STR=$(busybox date -d "@${EPOCH}" +"\033[1;30m%Y-%m-%dT\033[0;37m%H:%M:%S\033[1;30m%z")
-  # +0000 -> +00:00
-  echo "${TIME_STR:0:-2}:${TIME_STR:0-2}\033[0m"
 }
 
 _log_formatter() {
   local TARGET_LEVEL="${LOG_LEVEL:-}";
   local LEVEL="${1}";
-  local EPOCH="${2}";
+  local TIME_WITH_COLOR="${2}";
   local LOCATION="${3}";
   local SCOPE="${4}";
   TARGET_LEVEL=$(_log_level_to_upper "${TARGET_LEVEL}")
   LEVEL=$(_log_level_to_upper "${LEVEL}")
-  _log_skip "${TARGET_LEVEL}" "${LEVEL}" && return 0;
+  local LEVEL_COLOR=
+  LEVEL_COLOR=$(_log_skip_echo_color "${TARGET_LEVEL}" "${LEVEL}") && return 0;
   shift 4;
-  local NO_COLOR='\033[0m'
-  local DGRAY='\033[1;30m'
-  local TIME_STR LOC_STR LEVEL_STR SCOPE_STR
-  TIME_STR="${DGRAY}[$(_color_iso_time "${EPOCH}")${DGRAY}]${NO_COLOR}"
+  # Faster for not using local variables. (tested in a micro benchmark)
+  # local DGRAY='\033[1;30m'
+  # local NO_COLOR='\033[0m'
+  # Formatting time logically should be done inside this function.
+  # But we let caller do it to reduce the number of calls of "date" to increase performance.
+  local TIME_STR="\033[1;30m[${TIME_WITH_COLOR}\033[1;30m]\033[0m"
+  local LEVEL_STR="${LEVEL_COLOR}[${LEVEL}]\033[0m "
+  local LOC_STR SCOPE_STR
   if [ -n "${LOCATION}" ]; then
-    LOC_STR="${DGRAY}[${LOCATION}]${NO_COLOR}"
+    LOC_STR="\033[1;30m[${LOCATION}]\033[0m"
   fi
-  LEVEL_STR="$(_level_color "${LEVEL}")[${LEVEL}]${NO_COLOR} "
   if [ -n "${SCOPE}" ]; then
-    SCOPE_STR="${DGRAY}${SCOPE}:${NO_COLOR} "
+    SCOPE_STR="\033[1;30m${SCOPE}:\033[0m "
   fi
-  local MSG_STR=
-  MSG_STR=$(echo "${*}" | tr '\n' ' ')
-  echo -e "${TIME_STR}${LOC_STR}${LEVEL_STR}${SCOPE_STR}${MSG_STR}" >&2
+  # sed to remove \n
+  # :a - Creates a label a for looping.
+  # N - Appends the next line to the pattern space.
+  # $!ba - Loops back to the label a if not the last line ($! means "not last line").
+  # s/\n/ /g - Substitutes all newline characters with a space.
+  echo -e "${TIME_STR}${LOC_STR}${LEVEL_STR}${SCOPE_STR}${*}" | sed ':a;N;$!ba;s/\n/ /g' >&2
+  # Here are a few alternatives to "sed"
+  # "echo without quotes" remove carriage returns, tabs and multiple spaces.
+  # "echo" is faster than "tr", but it does not preserve the leading space.
+  # That is why we don't use "echo" here.
+  # "tr '\n' ' '" is slow and adds a space to the end of the string.
+}
+
+_time_format() {
+  # To mimik format from "date -Isecond".
+  # Highlight time within the day in ISO-8601 (2024-11-23T21:50:13-08:00)
+  # local DGRAY="\033[1;30m"
+  # local LGRAY="\033[0;37m"
+  # local NO_COLOR='\033[0m'
+  # echo "${DGRAY}%Y-%m-%dT${LGRAY}%H:%M:%S${DGRAY}%z${NO_COLOR}"
+  # The following is faster than the above for not using local variables. (tested in a micro benchmark)
+  # Busybox date does not support %:z, only %z. So the time zone will be -0800.
+  # SC2028 (info): echo may not expand escape sequences. Use printf.
+  # shellcheck disable=SC2028
+  echo "\033[1;30m%Y-%m-%dT\033[0;37m%H:%M:%S\033[1;30m%z\033[0m"
 }
 
 # We want to print an empty line for log without an argument. Thus we do not run the following check.
@@ -216,38 +225,52 @@ log() {
     LEVEL="${1}";
     shift;
   fi
-  local EPOCH=
-  EPOCH="$(date +%s)"
-  _log_formatter "${LEVEL}" "${EPOCH}" "${LOCAL_NODE}" "${LOCAL_SCOPE}" "${@}";
+  local TIME_WITH_COLOR=
+  TIME_WITH_COLOR="$(date +"$(_time_format)")"
+  _log_formatter "${LEVEL}" "${TIME_WITH_COLOR}" "${LOCAL_NODE}" "${LOCAL_SCOPE}" "${@}";
 }
 
-_log_docker_time_epoch() {
-  # Convert timestamps from `docker service logs` to EPOCH.
+_log_docker_time() {
+  # Convert timestamps from `docker service logs`.
   # The timestamps is in UTC.
   # docker service logs --timestamps --no-task-ids <service>
   # 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg with spaces>
   local TIME_INPUT="${1}"
-  if ! busybox date -d "${TIME_INPUT}" -D "%Y-%m-%dT%H:%M:%S" -u +%s 2>/dev/null; then
-    date +%s
-    return 1
+  # We are expecting most inputs are correct.
+  # coreutils date can do the conversion in one command, thus faster.
+  # busybox date does not read timezone via "-d".
+  # date -d "${TIME_INPUT}" +"$(_time_format)" 2>/dev/null && return 0
+  local EPOCH=
+  if EPOCH=$(busybox date -d "${TIME_INPUT}" -D "%Y-%m-%dT%H:%M:%S" -u +%s 2>/dev/null); then
+    date -d "@${EPOCH}" +"$(_time_format)" 2>&1
+    return 0
+  fi
+  if [ -n "${TIME_INPUT}" ]; then
+    echo "${TIME_INPUT}"
+  else
+    date +"$(_time_format)"
   fi
+  return 1
 }
 
 # Convert logs from `docker service logs` to `log` format.
 # docker service logs --timestamps --no-task-ids <service>
 # 2023-06-22T01:20:54.535860111Z <task>@<node>    | <msg with spaces>
 _log_docker_line() {
-  local LOCAL_NODE="${NODE_NAME:-}"
-  local LOCAL_SCOPE="${LOG_SCOPE:-}"
+  local NODE="${NODE_NAME:-}"
+  local SCOPE="${LOG_SCOPE:-}"
   local LEVEL="INFO";
-  local EPOCH NODE SCOPE MESSAGE
   # Using the same regexp for all 4 parts:
-  local TIME_DOCKER TASK_DOCKER NODE_DOCKER
-  read -r TIME_DOCKER TASK_DOCKER NODE_DOCKER MESSAGE < <(echo "${*}" | sed -n -E "s/^(\S+) +(\S+)@(\S+) +\| ?/\1 \2 \3 /p");
-  EPOCH=$(_log_docker_time_epoch "${TIME_DOCKER}");
+  local TIME_DOCKER TASK_DOCKER NODE_DOCKER MESSAGE
+  # Add a "+" before the last part to ensure we preserve the leading spaces in the message.
+  read -r TIME_DOCKER TASK_DOCKER NODE_DOCKER MESSAGE < <(echo "${*}" | sed -n -E "s/^(\S+) +(\S+)@(\S+) +\| ?/\1 \2 \3 +/p");
+  local TIME_WITH_COLOR=
+  TIME_WITH_COLOR=$(_log_docker_time "${TIME_DOCKER}");
   if [ -n "${TASK_DOCKER}" ] || [ -n "${NODE_DOCKER}" ] || [ -n "${MESSAGE}" ]; then
     NODE="${NODE_DOCKER}"
     SCOPE="${TASK_DOCKER}"
+    # Remove the extra "+" we added above for preserving the leading spaces.
+    MESSAGE="${MESSAGE:1}"
     if _first_word_is_level "${MESSAGE}"; then
       LEVEL=$(_get_first_word "${MESSAGE}");
       MESSAGE=$(extract_string "${MESSAGE}" ' ' 2-);
@@ -256,11 +279,9 @@ _log_docker_line() {
     # All three are empty, sed failure indicates errors.
     # format error, imply that we receive an error message from the "docker service logs" command.
     LEVEL="ERROR"
-    NODE="${LOCAL_NODE}"
-    SCOPE="${LOCAL_SCOPE}"
     MESSAGE="${*}"
   fi
-  _log_formatter "${LEVEL}" "${EPOCH}" "${NODE}" "${SCOPE}" "${MESSAGE}";
+  _log_formatter "${LEVEL}" "${TIME_WITH_COLOR}" "${NODE}" "${SCOPE}" "${MESSAGE}";
 }
 
 _log_docker_multiple_lines() {
@@ -441,15 +462,10 @@ _get_docker_command_name_arg() {
 }
 
 _get_docker_command_detach() {
-  if echo "${@}" | grep_q "--detach=false"; then
-    echo "false"
-  elif echo "${@}" | grep_q "--detach"; then
-    # assume we find --detach or --detach=true.
-    echo "true"
-  else
-    echo "false"
-  fi
-  return 0
+  echo "${@}" | grep_q "--detach=false" && return 1;
+  # assume we find --detach or --detach=true.
+  echo "${@}" | grep_q "--detach" && return 0;
+  return 1;
 }
 
 docker_service_logs() {
@@ -557,25 +573,35 @@ _all_tasks_reach_state() {
   return 0
 }
 
-# Usage: wait_service_state <SERVICE_NAME> <WANT_STATE>
-# Wait for the service, usually a global job or a replicated job,
-# to reach either running or complete state.
+# Usage: wait_service_state <SERVICE_NAME> [WANT_STATE] [timeout in seconds]
+# Wait for the service, usually a global job or a replicated job, to reach either running or complete state.
 # Valid WANT_STATE includes "Running" and "Complete"
-# When the WANT_STATE is complete, the function returns immediately
-# when any of the tasks of the service fails.
-# In case of task failing, the function returns a non-zero value.
+# When the WANT_STATE is complete, the function returns immediately when any of the tasks of the service fails.
+# In case of task failing, the function returns the first failing task's return value.
+# When the WANT_STATE is empty, this function reports the status of all tasks and then returns.
 wait_service_state() {
   local SERVICE_NAME="${1}";
   local WANT_STATE="${2}";
+  local TIMEOUT_SECONDS="${3}";
   local CHECK_FAILURES=false
   [ "${WANT_STATE}" = "Complete" ] && CHECK_FAILURES=true
   local SLEEP_SECONDS=1
-  local DOCKER_CMD_ERROR=1
+  local START_TIME=
+  START_TIME=$(date +%s)
   local RETURN_VALUE=0
+  local DOCKER_CMD_ERROR=1
   local STATES=
   while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do
     DOCKER_CMD_ERROR=0
     RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break
+    local SECONDS_ELAPSED=
+    if is_number "${TIMEOUT_SECONDS}" \
+        && SECONDS_ELAPSED=$(first_minus_second "$(date +%s)" "${START_TIME}") \
+        && [ "${SECONDS_ELAPSED}" -ge "${TIMEOUT_SECONDS}" ]; then
+      log ERROR "wait_service_state ${SERVICE_NAME} ${WANT_STATE} timeout after ${SECONDS_ELAPSED}s."
+      RETURN_VALUE=2
+      break
+    fi
     sleep "${SLEEP_SECONDS}"
     DOCKER_CMD_ERROR=1
   done
@@ -636,9 +662,7 @@ docker_global_job() {
 # A job could fail when using docker_replicated_job.
 docker_replicated_job() {
   local SERVICE_NAME=
-  local IS_DETACH=
   SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
-  IS_DETACH=$(_get_docker_command_detach "${@}")
   # Add "--detach" to work around https://github.com/docker/cli/issues/2979
   # The Docker CLI does not exit on failures.
   log INFO "Starting replicated-job ${SERVICE_NAME}."
@@ -648,7 +672,7 @@ docker_replicated_job() {
     return 1
   fi
   # If the command line does not contain '--detach', the function returns til the replicated job is complete.
-  if ! "${IS_DETACH}"; then
+  if ! _get_docker_command_detach "${@}"; then
     wait_service_state "${SERVICE_NAME}" "Complete" || return $?
   fi
   return 0
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index 18766a6..bd8ad0a 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -403,10 +403,10 @@ gantry_remove_images() {
   log INFO "Done removing images.";
 }
 
-_correct_service_name() {
+_sanitize_service_name() {
   local SERVICE_NAME="${1}"
   SERVICE_NAME=$(echo "${SERVICE_NAME}" | tr ' ' '-')
-  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME="${SERVICE_NAME:0:63}"
+  [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME="g${SERVICE_NAME:0-62}"
   echo "${SERVICE_NAME}"
 }
 
@@ -420,7 +420,7 @@ _remove_images() {
     return 0
   fi
   local SERVICE_NAME="${1:-"gantry-image-remover"}"
-  SERVICE_NAME=$(_correct_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_sanitize_service_name "${SERVICE_NAME}")
   docker_service_remove "${SERVICE_NAME}"
   local IMAGES_TO_REMOVE=
   IMAGES_TO_REMOVE=$(_static_variable_read_list STATIC_VAR_IMAGES_TO_REMOVE)
@@ -1013,14 +1013,18 @@ _update_single_service() {
   local UPDATE_COMMAND="${TIMEOUT_COMMAND} docker ${AUTH_CONFIG} service update"
   local UPDATE_RETURN_VALUE=0
   local UPDATE_MSG=
+  # Add "2>/dev/null" outside the $(cmd) to suppress the "Terminated" message from "busybox timeout".
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  UPDATE_MSG=$(${UPDATE_COMMAND} --quiet ${AUTOMATIC_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}" 2>&1);
+  UPDATE_MSG=$(${UPDATE_COMMAND} --quiet ${AUTOMATIC_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}" 2>&1) 2>/dev/null;
   UPDATE_RETURN_VALUE=$?
   if [ "${UPDATE_RETURN_VALUE}" != 0 ]; then
-    # https://git.savannah.gnu.org/cgit/coreutils.git/tree/src/timeout.c
+    # When there is a timeout:
+    # * coreutils timeout returns 124: https://git.savannah.gnu.org/cgit/coreutils.git/tree/src/timeout.c
+    # * busybox timeout returns 143
     local TIMEOUT_RETURN_CODE=124
+    timeout --help 2>&1 | grep_q_i "BusyBox" && TIMEOUT_RETURN_CODE=143
     local TIMEOUT_MSG=""
     if [ -n "${TIMEOUT_COMMAND}" ] && [ "${UPDATE_RETURN_VALUE}" = "${TIMEOUT_RETURN_CODE}" ]; then
       TIMEOUT_MSG="The return value ${UPDATE_RETURN_VALUE} indicates the job timed out."
diff --git a/tests/README.md b/tests/README.md
index b334374..bc72c2b 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -29,14 +29,11 @@ bash shellspec --jobs 50
 
 To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) installed):
 ```
-bash shellspec --kcov --tag coverage:true
+bash shellspec --kcov
 ```
 
 If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
 ```
 export GANTRY_TEST_CONTAINER_REPO_TAG=<gantry image>:<tag>
-bash shellspec --tag "container_test:true" "coverage:true"
+bash shellspec --jobs 50
 ```
-
-> NOTE: Negative tests will hang when testing a *Gantry* container, which may be due to a bug in shellspec. So when testing *Gantry* images, we should run only tests with tag `container_test:true`.
-
diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index 3189ed0..000aa2b 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -20,7 +20,7 @@ Describe 'cleanup-images'
   SUITE_NAME="cleanup-images"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_CLEANUP_IMAGES_false" "container_test:true" "coverage:true"
+  Describe "test_CLEANUP_IMAGES_false"
     TEST_NAME="test_CLEANUP_IMAGES_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -63,7 +63,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true" "coverage:true"
+  Describe "test_CLEANUP_IMAGES_OPTIONS_bad"
     TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -110,7 +110,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true" "coverage:true"
+  Describe "test_CLEANUP_IMAGES_OPTIONS_good"
     TEST_NAME="test_CLEANUP_IMAGES_OPTIONS_good"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -156,7 +156,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" "coverage:true"
+  Describe "test_IMAGES_TO_REMOVE_none_empty"
     # Test the remove image entrypoint. To improve coverage.
     TEST_NAME="test_IMAGES_TO_REMOVE_none_empty"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index f659d7d..0e8c70c 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -19,7 +19,7 @@ Describe 'common-options'
   SUITE_NAME="common-options"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_common_DOCKER_HOST_not_swarm_manager" "container_test:false" "coverage:true"
+  Describe "test_common_DOCKER_HOST_not_swarm_manager"
     TEST_NAME="test_common_DOCKER_HOST_not_swarm_manager"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -27,12 +27,8 @@ Describe 'common-options'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
-      export DOCKER_HOST="8.8.8.8:53"
-      local RETURN_VALUE=0
+      export GANTRY_TEST_DOCKER_HOST="8.8.8.8:53"
       run_gantry "${TEST_NAME}"
-      RETURN_VALUE="${?}"
-      export DOCKER_HOST=
-      return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -69,7 +65,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
   End
-  Describe "test_common_LOG_LEVEL_none" "container_test:true" "coverage:true"
+  Describe "test_common_LOG_LEVEL_none"
     TEST_NAME="test_common_LOG_LEVEL_none"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -93,10 +89,12 @@ Describe 'common-options'
     End
   End
   # Do not run test_common_no_new_env with the kcov, which alters the environment variables.
-  Describe "test_common_no_new_env" "container_test:false" "coverage:false"
+  Describe "test_common_no_new_env"
     # Check there is no new variable set,
     # to avoid errors like https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
-    # We don't need to run this test using containers because we check env on the host, while the container test set env inside the container.
+    #
+    # It makes no sense to run run this test using containers because we check env on the host, while the container test set env inside the container.
+    # But it should not failed with a container. We are just testing GANTRY_LOG_LEVEL=WARN.
     TEST_NAME="test_common_no_new_env"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -111,16 +109,16 @@ Describe 'common-options'
       reset_gantry_env "${SERVICE_NAME}"
       # There should be no warnings or errors. So it should work the same as LOG_LEVLE=NONE.
       export GANTRY_LOG_LEVEL=WARN
-
-      # Allow the following 3 mismatches used in log() function.
-      unset LOG_LEVEL NODE_NAME LOG_SCOPE
-      # _ contains the last command. declare is a bash builtin.
-      unset _; declare -p > "${ENV_BEFORE_RUN}"
+      declare -p > "${ENV_BEFORE_RUN}"
       run_gantry "${TEST_NAME}"
-      # Allow the following 3 mismatches used in log() function.
-      unset LOG_LEVEL NODE_NAME LOG_SCOPE
-      unset _; declare -p > "${ENV_AFTER_RUN}"
-      diff "${ENV_BEFORE_RUN}" "${ENV_AFTER_RUN}"
+      declare -p > "${ENV_AFTER_RUN}"
+      # Allow the 3 mismatches LOG_LEVEL NODE_NAME LOG_SCOPE used in log() function.
+      # Allow the 2 mismatches LINENO _ for kcov coverage.
+      for ALLOWED in LOG_LEVEL NODE_NAME LOG_SCOPE LINENO _; do
+        sed -i "s/^declare .* ${ALLOWED}=.*//" "${ENV_BEFORE_RUN}"
+        sed -i "s/^declare .* ${ALLOWED}=.*//" "${ENV_AFTER_RUN}"
+      done
+      diff --ignore-blank-lines "${ENV_BEFORE_RUN}" "${ENV_AFTER_RUN}"
       rm "${ENV_BEFORE_RUN}"
       rm "${ENV_AFTER_RUN}"
     }
@@ -135,7 +133,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message ".+"
     End
   End
-  Describe "test_common_PRE_POST_RUN_CMD" "container_test:true" "coverage:true"
+  Describe "test_common_PRE_POST_RUN_CMD"
     TEST_NAME="test_common_PRE_POST_RUN_CMD"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -148,7 +146,7 @@ Describe 'common-options'
       # Test that pre-run command can change the global configurations.
       export GANTRY_PRE_RUN_CMD="echo \"Pre update\"; GANTRY_UPDATE_OPTIONS=--detach=true; GANTRY_CLEANUP_IMAGES=false;"
       # This command outputs multiple lines.
-      local POST_CMD="for I in \$(seq 3 5); do echo \"OUTPUT_LINE=\$I\"; done"
+      local POST_CMD="for I in \$(seq 3 5); do echo \"TEST_OUTPUT_MULTIPLE_LINES=\$I\"; done"
       # Test that the command returns a non-zero value.
       export GANTRY_POST_RUN_CMD="echo \"Post update\"; ${POST_CMD}; false;"
       run_gantry "${TEST_NAME}"
@@ -162,8 +160,8 @@ Describe 'common-options'
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_message    "Pre update"
-      The stderr should satisfy spec_expect_message    "Finish pre-run command."
+      The stderr should satisfy spec_expect_message    "Pre update$"
+      The stderr should satisfy spec_expect_message    "Finish pre-run command.$"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -186,19 +184,16 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
-      The stderr should satisfy spec_expect_message    "Post update"
-      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=3"
-      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=4"
-      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=5"
-      The stderr should satisfy spec_expect_message    "Finish post-run command with a non-zero return value 1."
+      The stderr should satisfy spec_expect_message    "Post update$"
+      The stderr should satisfy spec_expect_message    "TEST_OUTPUT_MULTIPLE_LINES=3$"
+      The stderr should satisfy spec_expect_message    "TEST_OUTPUT_MULTIPLE_LINES=4$"
+      The stderr should satisfy spec_expect_message    "TEST_OUTPUT_MULTIPLE_LINES=5$"
+      The stderr should satisfy spec_expect_message    "Finish post-run command with a non-zero return value 1.$"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
   End
-  # run_gantry prints logs after gantry exists, while testing a container.
-  # In thes test, gantry never exit, but will be killed, thus there is no log.
-  # Therefore we disable the container test for this test.
-  Describe "test_common_SLEEP_SECONDS" "container_test:false" "coverage:true"
+  Describe "test_common_SLEEP_SECONDS"
     TEST_NAME="test_common_SLEEP_SECONDS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -207,9 +202,11 @@ Describe 'common-options'
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
       export GANTRY_SLEEP_SECONDS="7"
+      # Run run_gantry in background.
       run_gantry "${TEST_NAME}" &
       local PID="${!}"
       sleep $((GANTRY_SLEEP_SECONDS*3+1))
+      stop_gantry_container "${TEST_NAME}"
       kill "${PID}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -249,7 +246,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_message           "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
   End
-  Describe "test_common_SLEEP_SECONDS_not_a_number" "container_test:false" "coverage:true"
+  Describe "test_common_SLEEP_SECONDS_not_a_number"
     TEST_NAME="test_common_SLEEP_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index d14ebcb..0715d20 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -19,7 +19,7 @@ Describe 'filters'
   SUITE_NAME="filters"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_SERVICES_FILTERS_bad" "container_test:false" "coverage:true"
+  Describe "test_SERVICES_FILTERS_bad"
     TEST_NAME="test_SERVICES_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -64,7 +64,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true" "coverage:true"
+  Describe "test_SERVICES_EXCLUDED_multiple_services"
     TEST_NAME="test_SERVICES_EXCLUDED_multiple_services"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -122,7 +122,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_FILTERS_default" "container_test:true" "coverage:true"
+  Describe "test_SERVICES_EXCLUDED_FILTERS_default"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -173,7 +173,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
+  Describe "test_SERVICES_EXCLUDED_FILTERS_bad"
     TEST_NAME="test_SERVICES_EXCLUDED_FILTERS_bad"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_jobs_spec.sh b/tests/gantry_jobs_spec.sh
index 0bd26ae..0e447da 100644
--- a/tests/gantry_jobs_spec.sh
+++ b/tests/gantry_jobs_spec.sh
@@ -15,11 +15,11 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
-Describe 'update-jobs'
-  SUITE_NAME="update-jobs"
+Describe 'jobs'
+  SUITE_NAME="jobs"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_jobs_skipping" "container_test:true" "coverage:true"
+  Describe "test_jobs_skipping"
     # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
@@ -64,11 +64,11 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME_SUFFIX}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -86,7 +86,7 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+  Describe "test_jobs_UPDATE_JOBS_true"
     TEST_NAME="test_jobs_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -113,12 +113,12 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -137,7 +137,7 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+  Describe "test_jobs_label_UPDATE_JOBS_true"
     TEST_NAME="test_jobs_label_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -170,12 +170,12 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--detach=true.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -194,7 +194,7 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_jobs_no_running_tasks" "container_test:true" "coverage:true"
+  Describe "test_jobs_no_running_tasks"
     TEST_NAME="test_jobs_no_running_tasks"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -231,11 +231,11 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--replicas=0"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -253,4 +253,4 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-End # Describe 'update-jobs'
+End # Describe 'jobs'
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index d8a14db..09e9754 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -31,15 +31,18 @@
 # D=1 C=1 L=0 test_login_docker_config_no_label
 # D=1 C=1 L=1 test_login_docker_config_label_override SERVICE_NAME1
 
-Describe 'login_docker_config'
-  SUITE_NAME="login_docker_config"
+Describe 'login-docker-config'
+  SUITE_NAME="login-docker-config"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
-  Describe "test_login_docker_config_no_label" "container_test:true" "coverage:true"
+  Describe "test_login_docker_config_no_label"
     TEST_NAME="test_login_docker_config_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_no_label() {
       local TEST_NAME="${1}"
@@ -52,7 +55,7 @@ Describe 'login_docker_config'
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       reset_gantry_env "${SERVICE_NAME}"
-      export DOCKER_CONFIG="${CONFIG}"
+      export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
@@ -68,14 +71,14 @@ Describe 'login_docker_config'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -106,11 +109,14 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_docker_config_default_config" "container_test:true" "coverage:true"
+  Describe "test_login_docker_config_default_config"
     TEST_NAME="test_login_docker_config_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_default_config() {
       local TEST_NAME="${1}"
@@ -124,7 +130,7 @@ Describe 'login_docker_config'
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       # Do not set GANTRY_AUTH_CONFIG_LABEL on the service.
       reset_gantry_env "${SERVICE_NAME}"
-      export DOCKER_CONFIG="${CONFIG}"
+      export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       # Do not set GANTRY_REGISTRY_CONFIG to login to the default configuration.
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
@@ -141,7 +147,7 @@ Describe 'login_docker_config'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -177,14 +183,17 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
+  Describe "test_login_docker_config_label_override"
     TEST_NAME="test_login_docker_config_label_override"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME0="${SERVICE_NAME}-0"
     SERVICE_NAME1="${SERVICE_NAME}-1"
     SERVICE_NAME2="${SERVICE_NAME}-2"
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_start() {
       local TEST_NAME="${1}"
@@ -218,7 +227,7 @@ Describe 'login_docker_config'
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME1}"
       reset_gantry_env "${SERVICE_NAME}"
       # Inspection and updating should use DOCKER_CONFIG or the label.
-      export DOCKER_CONFIG="${CONFIG}"
+      export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
       # Set GANTRY_CLEANUP_IMAGES="false" to speedup the test. We are not testing removing image here.
       export GANTRY_CLEANUP_IMAGES="false"
@@ -241,21 +250,21 @@ Describe 'login_docker_config'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME0}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME2}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME2}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
@@ -296,4 +305,4 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-End # Describe 'login_docker_config'
+End # Describe 'login-docker-config'
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index 6db85d2..da6b932 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -15,15 +15,18 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
-Describe 'login_negative'
-  SUITE_NAME="login_negative"
+Describe 'login-negative'
+  SUITE_NAME="login-negative"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
-  Describe "test_login_no_login" "container_test:false" "coverage:true"
+  Describe "test_login_no_login"
     TEST_NAME="test_login_no_login"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_no_login() {
       local TEST_NAME="${1}"
@@ -34,7 +37,7 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_no_login "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_no_login "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -70,11 +73,14 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_incorrect_password" "container_test:false" "coverage:true"
+  Describe "test_login_incorrect_password"
     TEST_NAME="test_login_incorrect_password"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_incorrect_password() {
       local TEST_NAME="${1}"
@@ -104,14 +110,14 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_incorrect_password "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_incorrect_password "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -139,11 +145,11 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_read_only_file" "container_test:false" "coverage:true"
+  Describe "test_login_read_only_file"
     TEST_NAME="test_login_read_only_file"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(mktemp -d)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_read_only_file() {
       local TEST_NAME="${1}"
@@ -153,13 +159,16 @@ Describe 'login_negative'
       local USERNAME="${5}"
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      # Set the config folder to read only. (It won't work for container_test)
+      # When running with an image, we are not changing the folder inside the contianer.
+      # So do not run the test with a container/image.
       mkdir -p "${CONFIG}"
       chmod 444 "${CONFIG}"
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
       reset_gantry_env "${SERVICE_NAME}"
+      # Use GANTRY_TEST_HOST_TO_CONTAINER to mount the file from host to the container.
+      export GANTRY_TEST_HOST_TO_CONTAINER="${CONFIG}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
@@ -169,20 +178,20 @@ Describe 'login_negative'
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
-      [ -d "${CONFIG}" ] && chmod 777 "${CONFIG}" && rm -r "${CONFIG}"
+      # [ -d "${CONFIG}" ] && chmod 777 "${CONFIG}" && rm -r "${CONFIG}"
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_read_only_file "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_read_only_file "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${FAILED_TO_LOGIN_TO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -210,11 +219,14 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_config_mismatch_default" "container_test:false" "coverage:true"
+  Describe "test_login_config_mismatch_default"
     TEST_NAME="test_login_config_mismatch_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config_mismatch_default() {
       local TEST_NAME="${1}"
@@ -254,21 +266,21 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -295,11 +307,14 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_config_mismatch_no_default" "container_test:false" "coverage:true"
+  Describe "test_login_config_mismatch_no_default"
     TEST_NAME="test_login_config_mismatch_no_default"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config_mismatch_no_default() {
       local TEST_NAME="${1}"
@@ -332,22 +347,22 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       # This message does not present, because we don't login with the default configuration.
       The stderr should satisfy spec_expect_no_message "${USER_LOGGED_INTO_DEFAULT}.*"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -374,7 +389,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_multi_services_no_label" "container_test:false" "coverage:true"
+  Describe "test_login_multi_services_no_label"
     # To test https://github.com/shizunge/gantry/issues/64#issuecomment-2475499085
     TEST_NAME="test_login_multi_services_no_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
@@ -383,7 +398,10 @@ Describe 'login_negative'
     IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
     SERVICE_NAME0="${SERVICE_NAME}-0"
     SERVICE_NAME1="${SERVICE_NAME}-1"
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_start() {
       local TEST_NAME="${1}"
@@ -446,16 +464,16 @@ Describe 'login_negative'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_multi_services_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_multi_services_no_label "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME0}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME1}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME0}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME1}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
@@ -489,11 +507,14 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format" "container_test:false" "coverage:true"
+  Describe "test_login_REGISTRY_CONFIGS_FILE_bad_format"
     TEST_NAME="test_login_REGISTRY_CONFIGS_FILE_bad_format"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_REGISTRY_CONFIGS_FILE_bad_format() {
       local TEST_NAME="${1}"
@@ -522,7 +543,7 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_REGISTRY_CONFIGS_FILE_bad_format "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -530,7 +551,7 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "format error.*Found extra item\(s\)"
       The stderr should satisfy spec_expect_message    "format error.*Missing item\(s\)"
-      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}"
@@ -559,11 +580,14 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_file_not_exist" "container_test:false" "coverage:true"
+  Describe "test_login_file_not_exist"
     TEST_NAME="test_login_file_not_exist"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_file_not_exist() {
       local TEST_NAME="${1}"
@@ -590,13 +614,13 @@ Describe 'login_negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_file_not_exist "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING_ALL}.*${SKIP_REASON_PREVIOUS_ERRORS}"
@@ -625,4 +649,4 @@ Describe 'login_negative'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-End # Describe 'login_negative'
+End # Describe 'login-negative'
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 4c26906..c6d02ed 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -22,11 +22,14 @@ Describe 'login'
   SUITE_NAME="login"
   BeforeAll "initialize_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
   AfterAll "finish_all_tests ${SUITE_NAME} ENFORCE_LOGIN"
-  Describe "test_login_config" "container_test:true" "coverage:true"
+  Describe "test_login_config"
     TEST_NAME="test_login_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config() {
       local TEST_NAME="${1}"
@@ -58,14 +61,14 @@ Describe 'login'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -74,7 +77,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME}"
       # 2 "--with-registry-auth" for SERVICE_NAME. One is automatically added. The other is from user.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*automatically.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*specified by user.*${SERVICE_NAME}"
@@ -96,11 +99,11 @@ Describe 'login'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_default_config" "container_test:true" "coverage:true"
+  Describe "test_login_default_config"
     TEST_NAME="test_login_default_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="NotUsed"
+    AUTH_CONFIG="NotUsed"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_default_config() {
       local TEST_NAME="${1}"
@@ -130,7 +133,7 @@ Describe 'login'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_default_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -166,11 +169,14 @@ Describe 'login'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_REGISTRY_CONFIGS_FILE" "container_test:true" "coverage:true"
+  Describe "test_login_REGISTRY_CONFIGS_FILE"
     TEST_NAME="test_login_REGISTRY_CONFIGS_FILE"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="C$(unique_id)"
+    # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
+    # Using a relative path, this the container will not write to the folder on the host.
+    # So do not use an absolute path, otherwise we cannot remove this folder on the host.
+    AUTH_CONFIG="C$(unique_id)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_REGISTRY_CONFIGS_FILE() {
       local TEST_NAME="${1}"
@@ -204,14 +210,14 @@ Describe 'login'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_REGISTRY_CONFIGS_FILE "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${DEFAULT_CONFIGURATION}"
-      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${CONFIG}"
+      The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
       The stderr should satisfy spec_expect_no_message "${CONFIG_IS_NOT_A_DIRECTORY}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
@@ -220,7 +226,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME}"
       # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
@@ -241,11 +247,11 @@ Describe 'login'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_login_external_config" "container_test:true" "coverage:true"
+  Describe "test_login_external_config"
     TEST_NAME="test_login_external_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    CONFIG="$(mktemp -d)"
+    AUTH_CONFIG="$(mktemp -d)"
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_external_config() {
       local TEST_NAME="${1}"
@@ -260,7 +266,7 @@ Describe 'login'
       chmod 555 "${CONFIG}"
       # Do not set GANTRY_AUTH_CONFIG_LABEL on service.
       reset_gantry_env "${SERVICE_NAME}"
-      export DOCKER_CONFIG="${CONFIG}"
+      export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       # Use manifest to avoid write to DOCKER_CONFIG.
       # When running container test, Gantry runs as root.
       # We cannot remove DOCKER_CONFIG when it containes data from root.
@@ -276,7 +282,7 @@ Describe 'login'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_external_config "${TEST_NAME}" "${SERVICE_NAME}" "${CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_external_config "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index d79ca32..e7152ba 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -19,7 +19,7 @@ Describe 'manifest-command'
   SUITE_NAME="manifest-command"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_MANIFEST_CMD_none" "container_test:true" "coverage:true"
+  Describe "test_MANIFEST_CMD_none"
     TEST_NAME="test_MANIFEST_CMD_none"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -70,7 +70,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_MANIFEST_CMD_none_SERVICES_SELF" "container_test:true" "coverage:true"
+  Describe "test_MANIFEST_CMD_none_SERVICES_SELF"
     TEST_NAME="test_MANIFEST_CMD_none_SERVICES_SELF"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -118,7 +118,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_MANIFEST_CMD_manifest" "container_test:true" "coverage:true"
+  Describe "test_MANIFEST_CMD_manifest"
     TEST_NAME="test_MANIFEST_CMD_manifest"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -163,7 +163,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_MANIFEST_CMD_label" "container_test:true" "coverage:true"
+  Describe "test_MANIFEST_CMD_label"
     TEST_NAME="test_MANIFEST_CMD_label"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -213,7 +213,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_MANIFEST_CMD_unsupported_cmd" "container_test:false" "coverage:true"
+  Describe "test_MANIFEST_CMD_unsupported_cmd"
     TEST_NAME="test_MANIFEST_CMD_unsupported_cmd"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -260,7 +260,7 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_MANIFEST_CMD_failure" "container_test:false" "coverage:true"
+  Describe "test_MANIFEST_CMD_failure"
     TEST_NAME="test_MANIFEST_CMD_failure"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index 02dacdd..a3556f9 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -69,7 +69,7 @@ Describe 'notify'
   SUITE_NAME="notify"
   BeforeAll "_notify_before_all ${SUITE_NAME}"
   AfterAll "_notify_after_all ${SUITE_NAME}"
-  Describe "test_notify_apprise" "container_test:true" "coverage:true"
+  Describe "test_notify_apprise"
     TEST_NAME="test_notify_apprise"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -120,7 +120,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
   End
-  Describe "test_notify_apprise_no_new_image" "container_test:true" "coverage:true"
+  Describe "test_notify_apprise_no_new_image"
     TEST_NAME="test_notify_apprise_no_new_image"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -172,7 +172,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
   End
-  Describe "test_notify_apprise_bad_url" "container_test:true" "coverage:true"
+  Describe "test_notify_apprise_bad_url"
     TEST_NAME="test_notify_apprise_bad_url"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -217,7 +217,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "Failed to send notification via Apprise"
     End
   End
-  Describe "test_notify_on_change_new_image" "container_test:true" "coverage:true"
+  Describe "test_notify_on_change_new_image"
     TEST_NAME="test_notify_on_change_new_image"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -269,7 +269,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SEND_NOTIFY_APPRISE}"
     End
   End
-  Describe "test_notify_on_change_no_updates" "container_test:true" "coverage:true"
+  Describe "test_notify_on_change_no_updates"
     TEST_NAME="test_notify_on_change_no_updates"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -320,7 +320,7 @@ Describe 'notify'
       The stderr should satisfy spec_expect_message    "${SKIP_SENDING_NOTIFICATION}"
     End
   End
-  Describe "test_notify_on_change_errors" "container_test:false" "coverage:true"
+  Describe "test_notify_on_change_errors"
     TEST_NAME="test_notify_on_change_errors"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index b1f77ab..7e8dd4b 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -19,7 +19,7 @@ Describe 'service-parallel'
   SUITE_NAME="service-parallel"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_parallel_less_workers" "container_test:true" "coverage:true"
+  Describe "test_parallel_less_workers"
     TEST_NAME="test_parallel_less_workers"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -85,7 +85,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_parallel_more_workers" "container_test:true" "coverage:true"
+  Describe "test_parallel_more_workers"
     TEST_NAME="test_parallel_more_workers"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -132,7 +132,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
+  Describe "test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number"
     TEST_NAME="test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -176,7 +176,7 @@ Describe 'service-parallel'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number" "container_test:false" "coverage:true"
+  Describe "test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number"
     TEST_NAME="test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 0e561bf..81288c3 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -19,7 +19,7 @@ Describe 'rollback'
   SUITE_NAME="rollback"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_rollback_due_to_timeout" "container_test:false" "coverage:true"
+  Describe "test_rollback_due_to_timeout"
     TEST_NAME="test_rollback_due_to_timeout"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -44,12 +44,12 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
@@ -68,7 +68,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_rollback_failed" "container_test:false" "coverage:true"
+  Describe "test_rollback_failed"
     TEST_NAME="test_rollback_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -96,12 +96,12 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--with-registry-auth.*${SERVICE_NAME}"
@@ -120,7 +120,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_rollback_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
+  Describe "test_rollback_ROLLBACK_ON_FAILURE_false"
     TEST_NAME="test_rollback_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -146,12 +146,12 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
@@ -170,7 +170,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_rollback_label_failed" "container_test:false" "coverage:true"
+  Describe "test_rollback_label_failed"
     TEST_NAME="test_rollback_label_failed"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -201,12 +201,12 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--with-registry-auth.*${SERVICE_NAME}"
@@ -225,7 +225,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_rollback_label_ROLLBACK_ON_FAILURE_false" "container_test:false" "coverage:true"
+  Describe "test_rollback_label_ROLLBACK_ON_FAILURE_false"
     TEST_NAME="test_rollback_label_ROLLBACK_ON_FAILURE_false"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -253,12 +253,12 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index 1da1dfa..80ea6c3 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -19,7 +19,7 @@ Describe 'service-multiple-services'
   SUITE_NAME="service-multiple-services"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_multiple_services_excluded_filters" "container_test:true" "coverage:true"
+  Describe "test_multiple_services_excluded_filters"
     TEST_NAME="test_multiple_services_excluded_filters"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index f39be20..0ddd2ae 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -19,7 +19,7 @@ Describe "service-no-running-tasks"
   SUITE_NAME="service-no-running-tasks"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_no_running_tasks_replicated" "container_test:true" "coverage:true"
+  Describe "test_no_running_tasks_replicated"
     # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
@@ -99,7 +99,7 @@ Describe "service-no-running-tasks"
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_no_running_tasks_global" "container_test:true" "coverage:true"
+  Describe "test_no_running_tasks_global"
     TEST_NAME="test_no_running_tasks_global"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index 9daeb2b..492ec7a 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -19,7 +19,7 @@ Describe 'service-single-service'
   SUITE_NAME="service-single-service"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_new_image_no" "container_test:true" "coverage:true"
+  Describe "test_new_image_no"
     TEST_NAME="test_new_image_no"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -40,12 +40,13 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_CURRENT_IS_LATEST}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
@@ -65,7 +66,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
-  Describe "test_new_image_yes" "container_test:true" "coverage:true"
+  Describe "test_new_image_yes"
     TEST_NAME="test_new_image_yes"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -86,12 +87,13 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
@@ -111,7 +113,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
-  Describe "test_new_image_no_digest" "container_test:true" "coverage:true"
+  Describe "test_new_image_no_digest"
     TEST_NAME="test_new_image_no_digest"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -142,12 +144,13 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_DIGEST_IS_EMPTY}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
@@ -168,7 +171,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
-  Describe "test_new_image_SERVICES_SELF" "container_test:true" "coverage:true"
+  Describe "test_new_image_SERVICES_SELF"
     TEST_NAME="test_new_image_SERVICES_SELF"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -191,7 +194,6 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
@@ -199,6 +201,8 @@ Describe 'service-single-service'
       # We won't see the NUM_SERVICES_UPDATING message because GANTRY_SERVICES_SELF is not added to the list.
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
       The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 8d97030..ea47e37 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -25,7 +25,7 @@ Describe 'update-options'
   SUITE_NAME="update-options"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_update_UPDATE_OPTIONS" "container_test:true" "coverage:true"
+  Describe "test_update_UPDATE_OPTIONS"
     TEST_NAME="test_update_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -58,12 +58,12 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -81,7 +81,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_label_UPDATE_OPTIONS" "container_test:true" "coverage:true"
+  Describe "test_update_label_UPDATE_OPTIONS"
     TEST_NAME="test_update_label_UPDATE_OPTIONS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -117,12 +117,12 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -140,7 +140,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number" "container_test:false" "coverage:true"
+  Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -163,11 +163,11 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_message    "UPDATE_TIMEOUT_SECONDS ${MUST_BE_A_NUMBER}.*"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -185,7 +185,7 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_lable_UPDATE_TIMEOUT_SECONDS" "container_test:true" "coverage:true"
+  Describe "test_update_lable_UPDATE_TIMEOUT_SECONDS"
     TEST_NAME="test_update_lable_UPDATE_TIMEOUT_SECONDS"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
@@ -225,12 +225,12 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
-      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--label-add=gantry.test=${SERVICE_NAME}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${SET_TIMEOUT_TO} ${TIMEOUT}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 0d4be29..21a1dd1 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -15,8 +15,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
-set -a
-
 # Constant strings for checks.
 # START_WITHOUT_A_SQUARE_BRACKET ignores color codes. Use test_log not to trigger this check.
 export START_WITHOUT_A_SQUARE_BRACKET="^(?!(?:\x1b\[[0-9;]*[mG])?\[)"
@@ -46,7 +44,7 @@ export USER_LOGGED_INTO_DEFAULT="User logged in using the default Docker configu
 export ADDING_OPTIONS="Adding options"
 export ADDING_OPTIONS_WITH_REGISTRY_AUTH="Adding options.*--with-registry-auth"
 export SET_TIMEOUT_TO="Set timeout to"
-export RETURN_VALUE_INDICATES_TIMEOUT="The return value 124 indicates the job timed out."
+export RETURN_VALUE_INDICATES_TIMEOUT="The return value [0-9]+ indicates the job timed out."
 export THERE_ARE_ADDITIONAL_MESSAGES="There are additional messages from updating"
 export NUM_SERVICES_SKIP_JOBS="Skip updating [0-9]+ service\(s\) due to they are job\(s\)"
 export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)"
@@ -76,6 +74,8 @@ export TEST_SERVICE_IMAGE="alpine:latest"
 
 test_log() {
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^NONE$"  && return 0;
+  echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^ERROR$"  && return 0;
+  echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^WARN$"  && return 0;
   [ -n "${GANTRY_IMAGES_TO_REMOVE}" ] && echo "${*}" >&2 && return 0;
   echo "[$(date -Iseconds)] Test: ${*}" >&2
 }
@@ -144,14 +144,14 @@ common_setup_timeout() {
   local IMAGE_WITH_TAG="${2}"
   local SERVICE_NAME="${3}"
   local TIMEOUT="${4}"
-  local TIMEOUT_PLUS_ONE=$((TIMEOUT+1))
-  local TIMEOUT_PLUS_TWO=$((TIMEOUT+2))
+  local TIMEOUT_PLUS=$((TIMEOUT+1))
+  local TIMEOUT_MORE=$((TIMEOUT+2))
   initialize_test "${TEST_NAME}"
   # -1 thus the task runs forever.
   # exit will take longer than the timeout.
-  build_and_push_test_image "${IMAGE_WITH_TAG}" "-1" "${TIMEOUT_PLUS_TWO}"
+  build_and_push_test_image "${IMAGE_WITH_TAG}" "-1" "${TIMEOUT_MORE}"
   # Timeout set by "service create" should be smaller than the exit time above.
-  start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${TIMEOUT_PLUS_ONE}"
+  start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" "${TIMEOUT_PLUS}"
   build_and_push_test_image "${IMAGE_WITH_TAG}"
 }
 
@@ -420,8 +420,9 @@ initialize_test() {
 
 reset_gantry_env() {
   local SERVICE_NAME="${1}"
-  export DOCKER_CONFIG=
-  export DOCKER_HOST=
+  export GANTRY_TEST_HOST_TO_CONTAINER=
+  export GANTRY_TEST_DOCKER_CONFIG=
+  export GANTRY_TEST_DOCKER_HOST=
   export GANTRY_LOG_LEVEL="DEBUG"
   export GANTRY_NODE_NAME=
   export GANTRY_POST_RUN_CMD=
@@ -622,7 +623,7 @@ wait_zero_running_tasks() {
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
     # We do an extra step to to perform the exact match.
-    REPLICAS=$(echo "${REPLICAS}" | sed -n "s/\(.*\) ${SERVICE_NAME}$/\1/p")
+    REPLICAS=$(echo "${REPLICAS}" | sed -n -E "s/(.*) ${SERVICE_NAME}$/\1/p")
     if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
       echo "wait_zero_running_tasks Reach MAX_RETRIES ${MAX_RETRIES}" >&2
       return 1
@@ -665,7 +666,7 @@ pull_image_if_not_exist() {
 
 _enforce_login_enabled() {
   local ENFORCE_LOGIN="${1}"
-  test "${ENFORCE_LOGIN}" == "ENFORCE_LOGIN"
+  test "${ENFORCE_LOGIN}" = "ENFORCE_LOGIN"
 }
 
 _add_htpasswd() {
@@ -688,21 +689,13 @@ _add_htpasswd() {
 }
 
 _wait_service_state() {
-  local SERVICE_NAME="${1}"
-  local STATE="${2}"
-  local TRIES=0
-  local MAX_RETRIES=120
-  while ! docker service ps --format "{{.CurrentState}}" "${SERVICE_NAME}" | grep -q "${STATE}"; do
-    if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
-      echo "_wait_service_state Reach MAX_RETRIES ${MAX_RETRIES}" >&2
-      return 1
-    fi
-    TRIES=$((TRIES+1))
-    sleep 1
-  done
+  local SERVICE_NAME="${1}";
+  local WANT_STATE="${2}";
+  local TIMEOUT_SECONDS="${3}";
+  wait_service_state "${SERVICE_NAME}" "${WANT_STATE}" "${TIMEOUT_SECONDS}" 1>/dev/null 2>&1
 }
 
-_correct_test_service_name() {
+_sanitize_test_service_name() {
   local SERVICE_NAME="${1}"
   [ "${#SERVICE_NAME}" -gt 63 ] && SERVICE_NAME=${SERVICE_NAME:0:63}
   echo "${SERVICE_NAME}"
@@ -724,7 +717,7 @@ start_replicated_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
   local TIMEOUT_SECONDS="${3:-1}"
-  SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in replicated mode "
   # During creation:
   # * Add --detach to reduce the test runtime.
@@ -746,8 +739,8 @@ start_replicated_service() {
     $(_location_constraints) \
     --mode=replicated \
     --detach=true \
-    "${IMAGE_WITH_TAG}"
-  _wait_service_state "${SERVICE_NAME}" "Running"
+    "${IMAGE_WITH_TAG}";
+  _wait_service_state "${SERVICE_NAME}" "Running" 120
 }
 
 start_multiple_replicated_services() {
@@ -771,7 +764,7 @@ start_global_service() {
   local SERVICE_NAME="${1}"
   local IMAGE_WITH_TAG="${2}"
   local TIMEOUT_SECONDS="${3:-1}"
-  SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in global mode "
   # Do not add --detach, because we want to wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
@@ -794,7 +787,7 @@ _start_replicated_job() {
   local IMAGE_WITH_TAG="${2}"
   local TASK_SECONDS="${3:-1}"
   local EXIT_SECONDS="${4:-1}"
-  SERVICE_NAME=$(_correct_test_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
   echo "Creating service ${SERVICE_NAME} in replicated job mode "
   # Always add --detach=true, do not wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
@@ -808,9 +801,9 @@ _start_replicated_job() {
     $(_location_constraints) \
     --mode=replicated-job \
     --detach=true \
-    "${IMAGE_WITH_TAG}"
+    "${IMAGE_WITH_TAG}";
   # wait until the job is running
-  _wait_service_state "${SERVICE_NAME}" "Running"
+  _wait_service_state "${SERVICE_NAME}" "Running" 120
 }
 
 stop_service() {
@@ -867,6 +860,15 @@ _get_entrypoint() {
   echo "source ${STATIC_VAR_ENTRYPOINT}"
 }
 
+_get_file_readonly() {
+  local NAME="${1}"
+  if [ -w "${NAME}" ]; then
+    echo "false"
+  else
+    echo "true"
+  fi
+}
+
 _add_file_to_mount_options() {
   local MOUNT_OPTIONS="${1}"
   local HOST_PATH="${2}"
@@ -874,40 +876,63 @@ _add_file_to_mount_options() {
     # Use the absolute path inside the container.
     local TARGET=
     TARGET=$(readlink -f "${HOST_PATH}")
-    MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${HOST_PATH},target=${TARGET}"
+    local READONLY=
+    READONLY=$(_get_file_readonly "${HOST_PATH}")
+    MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${HOST_PATH},target=${TARGET},readonly=${READONLY}"
   fi
   echo "${MOUNT_OPTIONS}"
 }
 
-_run_gantry_container() {
+stop_gantry_container() {
   local STACK="${1}"
   local SUT_REPO_TAG=
   SUT_REPO_TAG="$(_get_sut_image)"
   if [ -z "${SUT_REPO_TAG}" ]; then
-    return 1
+    return 0;
   fi
+  local RETURN_VALUE=0
   local SERVICE_NAME=
-  SERVICE_NAME="gantry-test-SUT-$(unique_id)"
+  SERVICE_NAME="gantry-test-SUT-${STACK}"
+  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
+  local CMD_OUTPUT=
+  docker service logs --raw "${SERVICE_NAME}"
+  if ! CMD_OUTPUT=$(docker service rm "${SERVICE_NAME}" 2>&1); then
+    echo "Failed to remove service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2
+    RETURN_VALUE=1
+  fi
+  return "${RETURN_VALUE}"
+}
+
+_run_gantry_container() {
+  local STACK="${1}"
+  local SUT_REPO_TAG="${2}"
+  pull_image_if_not_exist "${SUT_REPO_TAG}"
+  local SERVICE_NAME=
+  SERVICE_NAME="gantry-test-SUT-${STACK}"
+  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
   docker service rm "${SERVICE_NAME}" >/dev/null 2>&1;
   local MOUNT_OPTIONS=
-  MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${DOCKER_CONFIG}")
+  MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_TEST_HOST_TO_CONTAINER}")
+  MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_TEST_DOCKER_CONFIG}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_CONFIG_FILE}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_CONFIGS_FILE}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_HOST_FILE}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_PASSWORD_FILE}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_REGISTRY_USER_FILE}")
   test_log "Starting SUT service ${SERVICE_NAME} with image ${SUT_REPO_TAG}."
+  test_log "MOUNT_OPTIONS=${MOUNT_OPTIONS}"
   local RETURN_VALUE=0
   local CMD_OUTPUT=
   # SC2086 (info): Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
   if ! CMD_OUTPUT=$(docker service create --name "${SERVICE_NAME}" \
+    --detach=true \
     --mode replicated-job --restart-condition=none --network host \
     --constraint "node.role==manager" \
     --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \
     ${MOUNT_OPTIONS} \
-    --env "DOCKER_CONFIG=${DOCKER_CONFIG}" \
-    --env "DOCKER_HOST=${DOCKER_HOST}" \
+    --env "DOCKER_CONFIG=${GANTRY_TEST_DOCKER_CONFIG}" \
+    --env "DOCKER_HOST=${GANTRY_TEST_DOCKER_HOST}" \
     --env "GANTRY_LOG_LEVEL=${GANTRY_LOG_LEVEL}" \
     --env "GANTRY_NODE_NAME=${GANTRY_NODE_NAME}" \
     --env "GANTRY_POST_RUN_CMD=${GANTRY_POST_RUN_CMD}" \
@@ -948,22 +973,53 @@ _run_gantry_container() {
     echo "Failed to create service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2
     RETURN_VALUE=1
   fi
-  docker service logs --raw "${SERVICE_NAME}"
-  if ! CMD_OUTPUT=$(docker service rm "${SERVICE_NAME}" 2>&1); then
-    echo "Failed to remove service ${SERVICE_NAME}: ${CMD_OUTPUT}" >&2
-    RETURN_VALUE=1
-  fi
+  _wait_service_state "${SERVICE_NAME}" "Complete" 120
+  RETURN_VALUE=$?
+  stop_gantry_container "${STACK}" || return 1
   return "${RETURN_VALUE}"
 }
 
+_restore_env() {
+  local ENV_NAME="${1}"
+  local ENV_SET="${2}"
+  local ENV_VALUE="${3}"
+  if [ "${ENV_SET}" = "1" ]; then
+    eval "export ${ENV_NAME}=\"${ENV_VALUE}\""
+  else
+    unset "${ENV_NAME}"
+  fi
+}
+
 run_gantry() {
   local STACK="${1}"
-  if _run_gantry_container "${STACK}"; then
-    return 0
+  local DOCKER_CONFIG_SET=0
+  local OLD_DOCKER_CONFIG=
+  local DOCKER_HOST_SET=0
+  local OLD_DOCKER_HOST=
+  if env | grep_q "^DOCKER_CONFIG="; then
+    DOCKER_CONFIG_SET=1
+    OLD_DOCKER_CONFIG="${DOCKER_CONFIG}"
+  fi
+  if env | grep_q "^DOCKER_HOST="; then
+    DOCKER_HOST_SET=1
+    OLD_DOCKER_HOST="${DOCKER_HOST}"
   fi
-  local ENTRYPOINT=
-  ENTRYPOINT=$(_get_entrypoint) || return 1
-  ${ENTRYPOINT} "${STACK}"
+  local RETURN_VALUE=1
+  local SUT_REPO_TAG=
+  SUT_REPO_TAG="$(_get_sut_image)"
+  if [ -n "${SUT_REPO_TAG}" ]; then
+    _run_gantry_container "${STACK}" "${SUT_REPO_TAG}"
+    RETURN_VALUE=$?
+  else
+    [ -n "${GANTRY_TEST_DOCKER_CONFIG}" ] && export DOCKER_CONFIG="${GANTRY_TEST_DOCKER_CONFIG}"
+    [ -n "${GANTRY_TEST_DOCKER_HOST}" ] && export DOCKER_HOST="${GANTRY_TEST_DOCKER_HOST}"
+    local ENTRYPOINT=
+    if ENTRYPOINT=$(_get_entrypoint); then
+      ${ENTRYPOINT} "${STACK}"
+      RETURN_VALUE=$?
+    fi
+  fi
+  _restore_env "DOCKER_CONFIG" "${DOCKER_CONFIG_SET}" "${OLD_DOCKER_CONFIG}"
+  _restore_env "DOCKER_HOST" "${DOCKER_HOST_SET}" "${OLD_DOCKER_HOST}"
+  return "${RETURN_VALUE}"
 }
-
-set +a

From 4fb8d71b51ecc774815db9a8f65d57ff15b638eb Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Wed, 27 Nov 2024 17:20:21 -0800
Subject: [PATCH 22/27] [docs] add a note about label on services. remove
 unnecessary comments from examples.

---
 README.md                           | 2 ++
 examples/cronjob/docker-compose.yml | 2 --
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 7049251..4990fd0 100644
--- a/README.md
+++ b/README.md
@@ -117,6 +117,8 @@ Labels can be added to services to modify the behavior of *Gantry* for particula
 | `gantry.update.options=<string>`         | Override [`GANTRY_UPDATE_OPTIONS`](#to-add-options-to-services-update). |
 | `gantry.update.timeout_seconds=<number>` | Override [`GANTRY_UPDATE_TIMEOUT_SECONDS`](#to-add-options-to-services-update). |
 
+> NOTE: You must apply the labels to the services not the containers. If you are using docker compose files to setup your services, you need to add the label to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section.
+
 ## FAQ
 
 [Authentication](docs/authentication.md)
diff --git a/examples/cronjob/docker-compose.yml b/examples/cronjob/docker-compose.yml
index b3ac1a0..89ccb76 100644
--- a/examples/cronjob/docker-compose.yml
+++ b/examples/cronjob/docker-compose.yml
@@ -7,8 +7,6 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
       - "GANTRY_NODE_NAME={{.Node.Hostname}}"
-      # The gantry service is able to find the name of itself service. Use GANTRY_SERVICES_SELF when you want to set a different value.
-      # - "GANTRY_SERVICES_SELF=${STACK}_gantry"
       - "GANTRY_SLEEP_SECONDS=0"
     deploy:
       replicas: 0

From 4a4d7c230ad33ee99cf1a4349ed3b26ce076cf7f Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Wed, 27 Nov 2024 17:20:31 -0800
Subject: [PATCH 23/27] [gantry] change some log level from debug to info.

---
 src/lib-common.sh                | 12 +++++---
 src/lib-gantry.sh                | 50 ++++++++++++++++----------------
 tests/spec_gantry_test_helper.sh |  2 +-
 3 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/src/lib-common.sh b/src/lib-common.sh
index c8d8bc5..5b5ada2 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -620,11 +620,13 @@ docker_service_remove() {
   local SERVICE_NAME="${1}"
   local POST_COMMAND="${2}"
   ! _docker_service_exists "${SERVICE_NAME}" && return 0
-  log INFO "Removing service ${SERVICE_NAME}."
+  log DEBUG "Removing service ${SERVICE_NAME}."
+  local RETURN_VALUE=0
   local LOG=
   if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then
+    RETURN_VALUE=$?
     log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}"
-    return 1
+    return "${RETURN_VALUE}"
   fi
   if [ -n "${POST_COMMAND}" ]; then
     eval "${POST_COMMAND}"
@@ -743,11 +745,13 @@ docker_remove() {
   if [ -z "${STATUS}" ]; then
     return 0
   fi
-  log INFO "Removing container ${CNAME}."
+  log DEBUG "Removing container ${CNAME}."
   if [ "${STATUS}" = "running" ]; then
     docker stop "${CNAME}" >/dev/null 2>/dev/null
   fi
-  docker rm "${CNAME}" >/dev/null
+  # If the container is created with "--rm", it will be removed automatically when being stopped.
+  docker rm -f "${CNAME}" >/dev/null;
+  log INFO "Removed container ${CNAME}."
 }
 
 docker_run() {
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index bd8ad0a..3f0a21a 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -66,7 +66,7 @@ _read_env_or_label() {
   local LABEL_VALUE=
   LABEL_VALUE=$(_get_label_from_service "${SERVICE_NAME}" "${LABEL}")
   if [ -n "${LABEL_VALUE}" ]; then
-    log DEBUG "Use value \"${LABEL_VALUE}\" from label ${LABEL} on the service ${SERVICE_NAME}."
+    log INFO "Use value \"${LABEL_VALUE}\" from label ${LABEL} on the service ${SERVICE_NAME}."
     echo "${LABEL_VALUE}"
     return 0
   fi
@@ -441,7 +441,7 @@ _remove_images() {
   log DEBUG "Set IMAGES_REMOVER=${IMAGES_REMOVER}"
   local IMAGES_TO_REMOVE_LIST=
   IMAGES_TO_REMOVE_LIST=$(echo "${IMAGES_TO_REMOVE}" | tr '\n' ' ')
-  [ -n "${CLEANUP_IMAGES_OPTIONS}" ] && log DEBUG "Adding options \"${CLEANUP_IMAGES_OPTIONS}\" to the global job ${SERVICE_NAME}."
+  [ -n "${CLEANUP_IMAGES_OPTIONS}" ] && log INFO "Adding options \"${CLEANUP_IMAGES_OPTIONS}\" to the global job ${SERVICE_NAME}."
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
   docker_global_job --name "${SERVICE_NAME}" \
@@ -550,7 +550,7 @@ _report_services() {
   case "${CONDITION}" in
     "on-change")
       if [ "${NUM_UPDATED}" = "0" ] && [ "${NUM_FAILED_PLUS_ERRORS}" = "0" ]; then
-        log DEBUG "There are no updates or errors for notification."
+        log INFO "There are no updates or errors for notification."
         SEND_NOTIFICATION="false"
       fi
       ;;
@@ -558,7 +558,7 @@ _report_services() {
       ;;
   esac
   if ! is_true "${SEND_NOTIFICATION}"; then
-    log DEBUG "Skip sending notification."
+    log INFO "Skip sending notification."
     return 0
   fi
   local TYPE="success"
@@ -632,7 +632,7 @@ _service_is_self() {
       GANTRY_SERVICES_SELF=$(gantry_current_service_name)
       export GANTRY_SERVICES_SELF
       _static_variable_add_unique_to_list STATIC_VAR_SERVICES_SELF "${GANTRY_SERVICES_SELF}"
-      [ -n "${GANTRY_SERVICES_SELF}" ] && log INFO "Set GANTRY_SERVICES_SELF to ${GANTRY_SERVICES_SELF}."
+      [ -n "${GANTRY_SERVICES_SELF}" ] && log DEBUG "Set GANTRY_SERVICES_SELF to ${GANTRY_SERVICES_SELF}."
     fi
   fi
   local SELF="${GANTRY_SERVICES_SELF}"
@@ -729,7 +729,7 @@ _skip_jobs() {
   fi
   local MODE=
   if MODE=$(_service_is_job "${SERVICE_NAME}"); then
-    log DEBUG "Skip updating ${SERVICE_NAME} because it is in ${MODE} mode."
+    log INFO "Skip updating ${SERVICE_NAME} because it is in ${MODE} mode."
     return 0
   fi
   return 1
@@ -743,18 +743,18 @@ _get_image_info() {
   local IMAGE="${3}"
   local AUTH_CONFIG=
   AUTH_CONFIG=$(_get_auth_config_from_service "${SERVICE_NAME}")
-  [ -n "${AUTH_CONFIG}" ] && log DEBUG "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
+  [ -n "${AUTH_CONFIG}" ] && log INFO "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
   local MSG=
   local RETURN_VALUE=0
   if echo "${MANIFEST_CMD}" | grep_q_i "buildx"; then
     # https://github.com/orgs/community/discussions/45779
-    [ -n "${MANIFEST_OPTIONS}" ] && log DEBUG "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker buildx imagetools inspect\"."
+    [ -n "${MANIFEST_OPTIONS}" ] && log INFO "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker buildx imagetools inspect\"."
     # SC2086: Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2086
     MSG=$(docker ${AUTH_CONFIG} buildx imagetools inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
     RETURN_VALUE=$?
   elif echo "${MANIFEST_CMD}" | grep_q_i "manifest"; then
-    [ -n "${MANIFEST_OPTIONS}" ] && log DEBUG "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker manifest inspect\"."
+    [ -n "${MANIFEST_OPTIONS}" ] && log INFO "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker manifest inspect\"."
     # SC2086: Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2086
     MSG=$(docker ${AUTH_CONFIG} manifest inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
@@ -795,7 +795,7 @@ _inspect_image() {
       # Always inspecting self, never skipping.
       MANIFEST_CMD="buildx"
     else
-      log DEBUG "Perform updating ${SERVICE_NAME} because MANIFEST_CMD is \"none\"."
+      log INFO "Perform updating ${SERVICE_NAME} because MANIFEST_CMD is \"none\"."
       echo "${IMAGE}"
       return 0
     fi
@@ -803,38 +803,38 @@ _inspect_image() {
   local NO_NEW_IMAGES=
   NO_NEW_IMAGES=$(_static_variable_read_list STATIC_VAR_NO_NEW_IMAGES)
   if _in_list "${NO_NEW_IMAGES}" "${DIGEST}"; then
-    log DEBUG "Skip updating ${SERVICE_NAME} because there is no known newer version of image ${IMAGE_WITH_DIGEST}."
+    log INFO "Skip updating ${SERVICE_NAME} because there is no known newer version of image ${IMAGE_WITH_DIGEST}."
     return 0
   fi
   local HAS_NEW_IMAGES=
   HAS_NEW_IMAGES=$(_static_variable_read_list STATIC_VAR_NEW_IMAGES)
   if _in_list "${HAS_NEW_IMAGES}" "${DIGEST}"; then
-    log DEBUG "Perform updating ${SERVICE_NAME} because there is a known newer version of image ${IMAGE_WITH_DIGEST}."
+    log INFO "Perform updating ${SERVICE_NAME} because there is a known newer version of image ${IMAGE_WITH_DIGEST}."
     echo "${IMAGE}"
     return 0
   fi
   local IMAGE_INFO=
   if ! IMAGE_INFO=$(_get_image_info "${SERVICE_NAME}" "${MANIFEST_CMD}" "${IMAGE}"); then
-    log DEBUG "Skip updating ${SERVICE_NAME} because there is a failure to obtain the manifest from the registry of image ${IMAGE}."
+    log INFO "Skip updating ${SERVICE_NAME} because there is a failure to obtain the manifest from the registry of image ${IMAGE}."
     return 1
   fi
-  [ -z "${IMAGE_INFO}" ] && log DEBUG "IMAGE_INFO is empty."
+  [ -z "${IMAGE_INFO}" ] && log WARN "IMAGE_INFO is empty for ${SERVICE_NAME}."
   if [ -z "${DIGEST}" ]; then
     # The image may not contain the digest for the following reasons:
     # 1. The image has not been push to or pulled from a V2 registry
     # 2. The image has been pulled from a V1 registry
     # 3. The service is updated without --with-registry-auth when registry requests authentication.
-    log DEBUG "Perform updating ${SERVICE_NAME} because DIGEST is empty in ${IMAGE_WITH_DIGEST}, assume there is a new image."
+    log INFO "Perform updating ${SERVICE_NAME} because DIGEST is empty in ${IMAGE_WITH_DIGEST}, assume there is a new image."
     echo "${IMAGE}"
     return 0
   fi
   if [ -n "${DIGEST}" ] && echo "${IMAGE_INFO}" | grep_q "${DIGEST}"; then
     _static_variable_add_unique_to_list STATIC_VAR_NO_NEW_IMAGES "${DIGEST}"
-    log DEBUG "Skip updating ${SERVICE_NAME} because the current version is the latest of image ${IMAGE_WITH_DIGEST}."
+    log INFO "Skip updating ${SERVICE_NAME} because the current version is the latest of image ${IMAGE_WITH_DIGEST}."
     return 0
   fi
   _static_variable_add_unique_to_list STATIC_VAR_NEW_IMAGES "${DIGEST}"
-  log DEBUG "Perform updating ${SERVICE_NAME} because there is a newer version of image ${IMAGE_WITH_DIGEST}."
+  log INFO "Perform updating ${SERVICE_NAME} because there is a newer version of image ${IMAGE_WITH_DIGEST}."
   echo "${IMAGE}"
   return 0
 }
@@ -950,8 +950,8 @@ _rollback_service() {
   # "service update --rollback" needs to take different options from "service update"
   local AUTOMATIC_OPTIONS=
   AUTOMATIC_OPTIONS=$(_get_service_rollback_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
-  [ -n "${AUTOMATIC_OPTIONS}" ] && log DEBUG "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
-  [ -n "${ROLLBACK_OPTIONS}" ] && log DEBUG "Adding options \"${ROLLBACK_OPTIONS}\" specified by user to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
+  [ -n "${AUTOMATIC_OPTIONS}" ] && log INFO "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
+  [ -n "${ROLLBACK_OPTIONS}" ] && log INFO "Adding options \"${ROLLBACK_OPTIONS}\" specified by user to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
   local ROLLBACK_MSG=
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
@@ -979,7 +979,7 @@ _get_timeout_command() {
   local TIMEOUT_COMMAND=""
   if [ "${UPDATE_TIMEOUT_SECONDS}" != "0" ]; then
     TIMEOUT_COMMAND="timeout ${UPDATE_TIMEOUT_SECONDS}"
-    log DEBUG "Set timeout to ${UPDATE_TIMEOUT_SECONDS} for updating ${SERVICE_NAME}."
+    log INFO "Set timeout to ${UPDATE_TIMEOUT_SECONDS} for updating ${SERVICE_NAME}."
   fi
   echo "${TIMEOUT_COMMAND}"
 }
@@ -1005,9 +1005,9 @@ _update_single_service() {
   local AUTOMATIC_OPTIONS=
   AUTH_CONFIG=$(_get_auth_config_from_service "${SERVICE_NAME}")
   AUTOMATIC_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
-  [ -n "${AUTH_CONFIG}" ] && log DEBUG "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
-  [ -n "${AUTOMATIC_OPTIONS}" ] && log DEBUG "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update\" for ${SERVICE_NAME}."
-  [ -n "${UPDATE_OPTIONS}" ] && log DEBUG "Adding options \"${UPDATE_OPTIONS}\" specified by user to the command \"docker service update\" for ${SERVICE_NAME}."
+  [ -n "${AUTH_CONFIG}" ] && log INFO "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
+  [ -n "${AUTOMATIC_OPTIONS}" ] && log INFO "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update\" for ${SERVICE_NAME}."
+  [ -n "${UPDATE_OPTIONS}" ] && log INFO "Adding options \"${UPDATE_OPTIONS}\" specified by user to the command \"docker service update\" for ${SERVICE_NAME}."
   local TIMEOUT_COMMAND=""
   TIMEOUT_COMMAND=$(_get_timeout_command "${SERVICE_NAME}") || return 1
   local UPDATE_COMMAND="${TIMEOUT_COMMAND} docker ${AUTH_CONFIG} service update"
@@ -1049,7 +1049,7 @@ _update_single_service() {
   fi
   _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATED "${SERVICE_NAME}"
   _static_variable_add_unique_to_list STATIC_VAR_IMAGES_TO_REMOVE "${PREVIOUS_IMAGE}"
-  log INFO "UPDATED ${SERVICE_NAME}. Use ${TIME_ELAPSED}."
+  log INFO "Updated ${SERVICE_NAME}. Use ${TIME_ELAPSED}."
   return 0
 }
 
@@ -1135,7 +1135,7 @@ gantry_get_services_list() {
   local S=
   for S in ${SERVICES} ; do
     if _in_list "${SERVICES_EXCLUDED}" "${S}" ; then
-      log DEBUG "Exclude service ${S} from updating."
+      log INFO "Exclude service ${S} from updating."
       continue
     fi
     # Add self to the first of the list.
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 21a1dd1..105ea23 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -51,7 +51,7 @@ export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)"
 export NUM_SERVICES_NO_NEW_IMAGES="No new images for [0-9]+ service\(s\)"
 export NUM_SERVICES_UPDATING="Updating [0-9]+ service\(s\)"
 export NO_UPDATES="No updates"
-export UPDATED="UPDATED"
+export UPDATED="Updated"
 export ROLLING_BACK="Rolling back"
 export FAILED_TO_ROLLBACK="Failed to roll back"
 export ROLLED_BACK="Rolled back"

From bc661f11106d7fbae3f8ee13c2fbf809e6804b84 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Wed, 27 Nov 2024 21:01:42 -0800
Subject: [PATCH 24/27] [tests] build a local gantry image to run container
 tests. Also used it as images remover.

---
 .github/workflows/coverage.yml                |  2 -
 .github/workflows/on-pull-request.yml         | 27 +++++-
 .github/workflows/on-push.yml                 | 80 ++++++-----------
 .github/workflows/on-release.yml              | 33 +++++--
 tests/README.md                               | 12 ++-
 tests/gantry_cleanup_images_spec.sh           | 16 ++--
 tests/gantry_common_options_spec.sh           | 24 ++---
 tests/gantry_filters_spec.sh                  | 16 ++--
 tests/gantry_jobs_spec.sh                     | 16 ++--
 tests/gantry_login_docker_config_spec.sh      | 12 +--
 tests/gantry_login_negative_spec.sh           | 32 +++----
 tests/gantry_login_spec.sh                    | 16 ++--
 tests/gantry_manifest_spec.sh                 | 24 ++---
 tests/gantry_notify_spec.sh                   | 24 ++---
 tests/gantry_parallel_spec.sh                 | 16 ++--
 tests/gantry_rollback_spec.sh                 | 20 ++---
 tests/gantry_service_multiple_spec.sh         |  4 +-
 tests/gantry_service_no_running_tasks_spec.sh |  8 +-
 tests/gantry_service_single_spec.sh           | 16 ++--
 tests/gantry_update_options_spec.sh           | 16 ++--
 tests/spec_gantry_test_helper.sh              | 89 ++++++++++++++-----
 21 files changed, 288 insertions(+), 215 deletions(-)

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 34d7321..cd9bdde 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -36,8 +36,6 @@ jobs:
         uses: actions/checkout@v4
       - name: Generate tests coverage reports
         run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
           bash shellspec --kcov --jobs 50
       - name: Upload reports
         uses: actions/upload-artifact@v4
diff --git a/.github/workflows/on-pull-request.yml b/.github/workflows/on-pull-request.yml
index 2e4bab3..e50ec18 100644
--- a/.github/workflows/on-pull-request.yml
+++ b/.github/workflows/on-pull-request.yml
@@ -27,8 +27,8 @@ jobs:
           shellcheck src/*.sh tests/*.sh
           set +e
 
-  tests:
-    name: Run tests
+  test_scripts:
+    name: Test scripts
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -66,3 +66,26 @@ jobs:
         run: |
           bash shellspec --pattern tests/${{ matrix.test_suit }}
 
+  test_container:
+    name: Test container
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.7.1
+      - name: Install shellspec
+        run: |
+          mkdir -p ~/shellspec
+          cd ~/shellspec
+          git clone https://github.com/shellspec/shellspec.git
+          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
+          echo -n "shellspec version: "
+          shellspec --version
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Run container tests
+        run: |
+          export GANTRY_TEST_CONTAINER=true
+          export GANTRY_TEST_CONTAINER_REPO_TAG=
+          echo "GANTRY_TEST_CONTAINER=${GANTRY_TEST_CONTAINER}"
+          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
+          bash shellspec --jobs 50
\ No newline at end of file
diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml
index 84ae731..39f4772 100644
--- a/.github/workflows/on-push.yml
+++ b/.github/workflows/on-push.yml
@@ -30,8 +30,8 @@ jobs:
           shellcheck src/*.sh tests/*.sh
           set +e
 
-  tests:
-    name: Test script
+  test_scripts:
+    name: Test scripts
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -67,17 +67,40 @@ jobs:
         uses: actions/checkout@v4
       - name: Run tests
         run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
           bash shellspec --pattern tests/${{ matrix.test_suit }}
 
+  test_container:
+    name: Test container
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.7.1
+      - name: Install shellspec
+        run: |
+          mkdir -p ~/shellspec
+          cd ~/shellspec
+          git clone https://github.com/shellspec/shellspec.git
+          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
+          echo -n "shellspec version: "
+          shellspec --version
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Run container tests
+        run: |
+          export GANTRY_TEST_CONTAINER=true
+          export GANTRY_TEST_CONTAINER_REPO_TAG=
+          echo "GANTRY_TEST_CONTAINER=${GANTRY_TEST_CONTAINER}"
+          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
+          bash shellspec --jobs 50
+
   build_and_push:
     name: Build and push Docker image
     runs-on: ubuntu-latest
     if: ${{ github.actor != 'dependabot[bot]' }}
     needs:
       - shellcheck
-      - tests
+      - test_scripts
+      - test_container
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
@@ -116,50 +139,3 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           provenance: false
-      - name: Extract tag
-        run: |
-          TAGS="${{ steps.meta.outputs.tags }}"
-          for TAG in ${TAGS}; do
-            if echo "${TAG}" | grep -q "ghcr.io/${{ github.repository }}-development:dev-"; then
-              echo "TAG=${TAG}"
-              echo "${TAG}" > tag.txt
-              break;   
-            fi
-          done
-      - name: Store tag
-        uses: actions/upload-artifact@v4
-        with:
-          name: tag
-          path: tag.txt
-          
-
-  container_tests:
-    name: Test container
-    runs-on: ubuntu-latest
-    if: ${{ github.actor != 'dependabot[bot]' }}
-    needs:
-      - build_and_push
-    steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3.7.1
-      - name: Install shellspec
-        run: |
-          mkdir -p ~/shellspec
-          cd ~/shellspec
-          git clone https://github.com/shellspec/shellspec.git
-          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
-          echo -n "shellspec version: "
-          shellspec --version
-      - name: Checkout Code
-        uses: actions/checkout@v4
-      - name: Load tag
-        uses: actions/download-artifact@v4
-        with:
-          name: tag
-      - name: Run tests
-        run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
-          export GANTRY_TEST_CONTAINER_REPO_TAG=$(cat tag.txt)
-          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
-          bash shellspec --jobs 50
diff --git a/.github/workflows/on-release.yml b/.github/workflows/on-release.yml
index be3b921..0e29c6f 100644
--- a/.github/workflows/on-release.yml
+++ b/.github/workflows/on-release.yml
@@ -24,8 +24,8 @@ jobs:
           shellcheck src/*.sh tests/*.sh
           set +e
 
-  tests:
-    name: Run tests
+  test_scripts:
+    name: Test scripts
     runs-on: ubuntu-latest
     steps:
       - name: Set up Docker Buildx
@@ -42,8 +42,30 @@ jobs:
         uses: actions/checkout@v4
       - name: Run tests
         run: |
-          export DOCKERHUB_PASSWORD=${{ secrets.DOCKERHUB_PASSWORD }}
-          export DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}
+          bash shellspec --jobs 50
+
+  test_container:
+    name: Test container
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3.7.1
+      - name: Install shellspec
+        run: |
+          mkdir -p ~/shellspec
+          cd ~/shellspec
+          git clone https://github.com/shellspec/shellspec.git
+          ln -s ~/shellspec/shellspec/shellspec /usr/local/bin/shellspec
+          echo -n "shellspec version: "
+          shellspec --version
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Run container tests
+        run: |
+          export GANTRY_TEST_CONTAINER=true
+          export GANTRY_TEST_CONTAINER_REPO_TAG=
+          echo "GANTRY_TEST_CONTAINER=${GANTRY_TEST_CONTAINER}"
+          echo "GANTRY_TEST_CONTAINER_REPO_TAG=${GANTRY_TEST_CONTAINER_REPO_TAG}"
           bash shellspec --jobs 50
 
   build_and_push:
@@ -51,7 +73,8 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - shellcheck
-      - tests
+      - test_scripts
+      - test_container
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
diff --git a/tests/README.md b/tests/README.md
index bc72c2b..3ab1302 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -32,7 +32,17 @@ To generate coverage (require [kcov](https://github.com/SimonKagstrom/kcov) inst
 bash shellspec --kcov
 ```
 
-If you want to test a container image of *Gantry*, you need to specify the image of *Gantry* via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
+The above commands test *Gantry* as a script running on the host directly. We also want to test *Gantry* running inside a container in case the environments are different between the host and the container.
+
+To test *Gantry* running inside a container, set the environment variable `GANTRY_TEST_CONTAINER` to `true`. The testing framework will build a local image of *Gantry*, then start a service of that image to run the test.
+
+```
+export GANTRY_TEST_CONTAINER=true
+bash shellspec --jobs 50
+```
+
+If you want to test a specific image of *Gantry*, you need to set the image of *Gantry* explicitly via the environment variable `GANTRY_TEST_CONTAINER_REPO_TAG`.
+
 ```
 export GANTRY_TEST_CONTAINER_REPO_TAG=<gantry image>:<tag>
 bash shellspec --jobs 50
diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
index 000aa2b..0c97367 100644
--- a/tests/gantry_cleanup_images_spec.sh
+++ b/tests/gantry_cleanup_images_spec.sh
@@ -27,9 +27,9 @@ Describe 'cleanup-images'
     test_CLEANUP_IMAGES_false() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_CLEANUP_IMAGES="false"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -70,11 +70,11 @@ Describe 'cleanup-images'
     test_CLEANUP_IMAGES_OPTIONS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_CLEANUP_IMAGES="true"
       # Image remover would fail due to the incorrect option.
       export GANTRY_CLEANUP_IMAGES_OPTIONS="--incorrect-option"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -117,10 +117,10 @@ Describe 'cleanup-images'
     test_CLEANUP_IMAGES_OPTIONS_good() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_CLEANUP_IMAGES="true"
       export GANTRY_CLEANUP_IMAGES_OPTIONS="--container-label=test"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -194,9 +194,9 @@ Describe 'cleanup-images'
       local IMAGE_WITH_TAG0="${IMAGE_WITH_TAG}-0"
       local IMAGE_WITH_TAG1="${IMAGE_WITH_TAG}-1"
       local IMAGE_WITH_TAG2="${IMAGE_WITH_TAG}-2"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_IMAGES_TO_REMOVE="${IMAGE_WITH_TAG0} ${IMAGE_WITH_TAG1} ${IMAGE_WITH_TAG2}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     test_end() {
       local TEST_NAME="${1}"
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index 0e8c70c..95680cb 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -26,9 +26,9 @@ Describe 'common-options'
     test_common_DOCKER_HOST_not_swarm_manager() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_HOST="8.8.8.8:53"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -72,10 +72,10 @@ Describe 'common-options'
     test_common_LOG_LEVEL_none() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Same as test_new_image_yes, except set LOG_LEVEL to NONE
       export GANTRY_LOG_LEVEL=NONE
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -106,11 +106,11 @@ Describe 'common-options'
       local ENV_AFTER_RUN=
       ENV_AFTER_RUN=$(mktemp)
 
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # There should be no warnings or errors. So it should work the same as LOG_LEVLE=NONE.
       export GANTRY_LOG_LEVEL=WARN
       declare -p > "${ENV_BEFORE_RUN}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       declare -p > "${ENV_AFTER_RUN}"
       # Allow the 3 mismatches LOG_LEVEL NODE_NAME LOG_SCOPE used in log() function.
       # Allow the 2 mismatches LINENO _ for kcov coverage.
@@ -140,7 +140,7 @@ Describe 'common-options'
     test_common_PRE_POST_RUN_CMD() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_OPTIONS=
       export GANTRY_CLEANUP_IMAGES=
       # Test that pre-run command can change the global configurations.
@@ -149,7 +149,7 @@ Describe 'common-options'
       local POST_CMD="for I in \$(seq 3 5); do echo \"TEST_OUTPUT_MULTIPLE_LINES=\$I\"; done"
       # Test that the command returns a non-zero value.
       export GANTRY_POST_RUN_CMD="echo \"Post update\"; ${POST_CMD}; false;"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -200,10 +200,10 @@ Describe 'common-options'
     test_common_SLEEP_SECONDS() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SLEEP_SECONDS="7"
       # Run run_gantry in background.
-      run_gantry "${TEST_NAME}" &
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}" &
       local PID="${!}"
       sleep $((GANTRY_SLEEP_SECONDS*3+1))
       stop_gantry_container "${TEST_NAME}"
@@ -253,9 +253,9 @@ Describe 'common-options'
     test_common_SLEEP_SECONDS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SLEEP_SECONDS="NotANumber"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
index 0715d20..006ee9f 100644
--- a/tests/gantry_filters_spec.sh
+++ b/tests/gantry_filters_spec.sh
@@ -26,9 +26,9 @@ Describe 'filters'
     test_SERVICES_FILTERS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SERVICES_FILTERS="BadFilterValue"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -75,7 +75,7 @@ Describe 'filters'
       local MAX_SERVICES_NUM="${3}"
       local NUM_SERVICES_EXCLUDED=$((MAX_SERVICES_NUM/2))
       local NUM_SERVICES_EXCLUDED_FILTER_START=$((NUM_SERVICES_EXCLUDED+1))
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       for NUM in $(seq 0 "${NUM_SERVICES_EXCLUDED}"); do
         local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
         export GANTRY_SERVICES_EXCLUDED="${GANTRY_SERVICES_EXCLUDED} ${SERVICE_NAME_NUM}"
@@ -86,7 +86,7 @@ Describe 'filters'
         docker_service_update --label-add "${LABEL}=true" "${SERVICE_NAME_NUM}"
       done
       export GANTRY_SERVICES_EXCLUDED_FILTERS="label=${LABEL}=true"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
     AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
@@ -131,14 +131,14 @@ Describe 'filters'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local MAX_SERVICES_NUM="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       local LABEL="gantry.services.excluded"
       for NUM in $(seq 0 "${MAX_SERVICES_NUM}"); do
         local SERVICE_NAME_NUM="${SERVICE_NAME}-${NUM}"
         docker_service_update --label-add "${LABEL}=true" "${SERVICE_NAME_NUM}"
       done
       # Do not set GANTRY_SERVICES_EXCLUDED_FILTERS, check the default one is working.
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
     AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
@@ -180,9 +180,9 @@ Describe 'filters'
     test_SERVICES_EXCLUDED_FILTERS_bad() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_SERVICES_EXCLUDED_FILTERS="BadFilterValue"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_jobs_spec.sh b/tests/gantry_jobs_spec.sh
index 0e447da..48a00a4 100644
--- a/tests/gantry_jobs_spec.sh
+++ b/tests/gantry_jobs_spec.sh
@@ -39,8 +39,8 @@ Describe 'jobs'
     test_jobs_skipping() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     test_end() {
       local TEST_NAME="${1}"
@@ -96,11 +96,11 @@ Describe 'jobs'
     test_jobs_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_JOBS="true"
       # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true"
       export GANTRY_UPDATE_OPTIONS="--detach=true"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -147,7 +147,7 @@ Describe 'jobs'
     test_jobs_label_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # label should override the global environment variable.
       export GANTRY_UPDATE_JOBS="false"
       local LABEL_AND_VALUE="gantry.update.jobs=true"
@@ -157,7 +157,7 @@ Describe 'jobs'
       # The job may not reach the desired "Complete" state and blocking update CLI. So add "--detach=true"
       LABEL_AND_VALUE="gantry.update.options=--detach=true"
       docker_service_update --detach=true --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -212,9 +212,9 @@ Describe 'jobs'
     test_jobs_no_running_tasks() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_JOBS="true"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     # The task will finish in ${TASK_SECONDS} seconds
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index 09e9754..0bd8ece 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -54,14 +54,14 @@ Describe 'login-docker-config'
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -129,14 +129,14 @@ Describe 'login-docker-config'
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       # Do not set GANTRY_AUTH_CONFIG_LABEL on the service.
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       # Do not set GANTRY_REGISTRY_CONFIG to login to the default configuration.
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       docker logout "${REGISTRY}" > /dev/null
       rm "${USER_FILE}"
@@ -225,14 +225,14 @@ Describe 'login-docker-config'
       # Inspection of SERVICE_NAME2 should pass (use DOCKER_CONFIG).
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME0}"
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME1}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Inspection and updating should use DOCKER_CONFIG or the label.
       export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
       # Set GANTRY_CLEANUP_IMAGES="false" to speedup the test. We are not testing removing image here.
       export GANTRY_CLEANUP_IMAGES="false"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${CONFIGS_FILE}"
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index da6b932..0af2a88 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -31,8 +31,8 @@ Describe 'login-negative'
     test_login_no_login() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -94,13 +94,13 @@ Describe 'login-negative'
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${INCORRECT_PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -166,7 +166,7 @@ Describe 'login-negative'
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Use GANTRY_TEST_HOST_TO_CONTAINER to mount the file from host to the container.
       export GANTRY_TEST_HOST_TO_CONTAINER="${CONFIG}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
@@ -174,7 +174,7 @@ Describe 'login-negative'
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -245,7 +245,7 @@ Describe 'login-negative'
       echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}"
       # The config name on the service is different from the config name used in GANTRY_REGISTRY_CONFIG
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Do not set GANTRY_REGISTRY_CONFIG, login to the default config.
       # export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
@@ -253,7 +253,7 @@ Describe 'login-negative'
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -329,14 +329,14 @@ Describe 'login-negative'
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       # The config name on the service is different from the config name used in GANTRY_REGISTRY_CONFIG
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -435,7 +435,7 @@ Describe 'login-negative'
       # Inspection of SERVICE_NAME0 should fail, because GANTRY_AUTH_CONFIG_LABEL is not found.
       # Inspection of SERVICE_NAME1 should pass, because configuration is set via GANTRY_AUTH_CONFIG_LABEL.
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME1}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
@@ -443,7 +443,7 @@ Describe 'login-negative'
       # Set GANTRY_CLEANUP_IMAGES="false" to speedup the test. We are not testing removing image here.
       export GANTRY_CLEANUP_IMAGES="false"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -531,10 +531,10 @@ Describe 'login-negative'
       # Missing an item from the line.
       echo "The-Only-Item-In-The-Line" >> "${CONFIGS_FILE}"
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${CONFIGS_FILE}"
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
@@ -599,14 +599,14 @@ Describe 'login-negative'
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
       local FILE_NOT_EXIST="/tmp/${CONFIG}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG_FILE="${FILE_NOT_EXIST}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${FILE_NOT_EXIST}"
       export GANTRY_REGISTRY_HOST_FILE="${FILE_NOT_EXIST}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${FILE_NOT_EXIST}"
       export GANTRY_REGISTRY_USER_FILE="${FILE_NOT_EXIST}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
       return "${RETURN_VALUE}"
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index c6d02ed..7aaf316 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -42,7 +42,7 @@ Describe 'login'
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
@@ -51,7 +51,7 @@ Describe 'login'
       # A duplicated "--with-registry-auth" will be added, but it should be ok.
       export GANTRY_UPDATE_OPTIONS="--with-registry-auth"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
@@ -116,13 +116,13 @@ Describe 'login'
       local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
       local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
       # Do not set GANTRY_AUTH_CONFIG_LABEL on the service.
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Do not set GANTRY_REGISTRY_CONFIG to login to the default configuration.
       export GANTRY_REGISTRY_HOST="${REGISTRY}"
       export GANTRY_REGISTRY_PASSWORD_FILE="${PASS_FILE}"
       export GANTRY_REGISTRY_USER_FILE="${USER_FILE}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       docker logout "${REGISTRY}" > /dev/null
       rm "${USER_FILE}"
@@ -191,7 +191,7 @@ Describe 'login'
       echo "# Test comments: CONFIG REGISTRY USERNAME PASSWORD" >> "${CONFIGS_FILE}"
       echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}"
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIGS_FILE="${CONFIGS_FILE}"
       # Since we pass credentials via the configs file, we can use other envs to login to docker hub and check the rate.
       # However we do not actually check whether we read rates correctly, in case password or usrename for docker hub is not set.
@@ -201,7 +201,7 @@ Describe 'login'
       export GANTRY_REGISTRY_PASSWORD="${DOCKERHUB_PASSWORD:-""}"
       export GANTRY_REGISTRY_USER="${DOCKERHUB_USERNAME:-""}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       rm "${CONFIGS_FILE}"
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
@@ -265,7 +265,7 @@ Describe 'login'
       echo "${PASSWORD}" | docker --config "${CONFIG}" login --username="${USERNAME}" --password-stdin "${REGISTRY}" > /dev/null 2>&1
       chmod 555 "${CONFIG}"
       # Do not set GANTRY_AUTH_CONFIG_LABEL on service.
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       # Use manifest to avoid write to DOCKER_CONFIG.
       # When running container test, Gantry runs as root.
@@ -274,7 +274,7 @@ Describe 'login'
       export GANTRY_MANIFEST_OPTIONS="--insecure"
       # Do not set --with-registry-auth to trigger a warning IMAGE_DIGEST_WARNING.
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       [ -d "${CONFIG}" ] && chmod 777 "${CONFIG}" && rm -r "${CONFIG}"
       return "${RETURN_VALUE}"
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index e7152ba..8fee3c2 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -26,10 +26,10 @@ Describe 'manifest-command'
     test_MANIFEST_CMD_none() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_CMD="none"
       export GANTRY_UPDATE_OPTIONS="--force"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -78,11 +78,11 @@ Describe 'manifest-command'
       # If the service is self, it will always run manifest checking. Even if the CMD is set to none
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Explicitly set GANTRY_SERVICES_SELF
       export GANTRY_SERVICES_SELF="${SERVICE_NAME}"
       export GANTRY_MANIFEST_CMD="none"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -125,10 +125,10 @@ Describe 'manifest-command'
     test_MANIFEST_CMD_manifest() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_OPTIONS="--insecure"
       export GANTRY_MANIFEST_CMD="manifest"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -170,7 +170,7 @@ Describe 'manifest-command'
     test_MANIFEST_CMD_label() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # The label should be equivalent to
       # export GANTRY_MANIFEST_CMD="manifest"
       # export GANTRY_MANIFEST_OPTIONS="--insecure"
@@ -178,7 +178,7 @@ Describe 'manifest-command'
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       LABEL_AND_VALUE="gantry.manifest.options=--insecure"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -220,10 +220,10 @@ Describe 'manifest-command'
     test_MANIFEST_CMD_unsupported_cmd() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_OPTIONS="--insecure"
       export GANTRY_MANIFEST_CMD="unsupported_cmd"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -278,8 +278,8 @@ Describe 'manifest-command'
     test_MANIFEST_CMD_failure() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index a3556f9..d535cb0 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -77,10 +77,10 @@ Describe 'notify'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local RETURN_VALUE=0
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       _print_and_cleanup_emails
       return "${RETURN_VALUE}"
@@ -128,10 +128,10 @@ Describe 'notify'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local RETURN_VALUE=0
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       _print_and_cleanup_emails
       return "${RETURN_VALUE}"
@@ -179,9 +179,9 @@ Describe 'notify'
     test_notify_apprise_bad_url() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://bad-url/notify"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -225,11 +225,11 @@ Describe 'notify'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local RETURN_VALUE=0
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_CONDITION="on-change"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       _print_and_cleanup_emails
       return "${RETURN_VALUE}"
@@ -277,10 +277,10 @@ Describe 'notify'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local RETURN_VALUE=0
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_NOTIFICATION_CONDITION="on-change"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       return "${RETURN_VALUE}"
     }
@@ -328,12 +328,12 @@ Describe 'notify'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local RETURN_VALUE=0
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_OPTIONS="--bad-options-that-causes-error"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_CONDITION="on-change"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       _print_and_cleanup_emails
       return "${RETURN_VALUE}"
diff --git a/tests/gantry_parallel_spec.sh b/tests/gantry_parallel_spec.sh
index 7e8dd4b..04fd2c6 100644
--- a/tests/gantry_parallel_spec.sh
+++ b/tests/gantry_parallel_spec.sh
@@ -41,10 +41,10 @@ Describe 'service-parallel'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local MAX_SERVICES_NUM="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_NUM_WORKERS="${MANIFEST_NUM_WORKERS}"
       export GANTRY_UPDATE_NUM_WORKERS=$((MAX_SERVICES_NUM/2+1))
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM} ${MAX_NO_NEW_IMAGE}"
     AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} $((MAX_SERVICES_NUM+MAX_NO_NEW_IMAGE))"
@@ -95,10 +95,10 @@ Describe 'service-parallel'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local MAX_SERVICES_NUM="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_NUM_WORKERS="${MANIFEST_NUM_WORKERS}"
       export GANTRY_UPDATE_NUM_WORKERS=$((MAX_SERVICES_NUM*3))
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
     AfterEach "common_cleanup_multiple ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${MAX_SERVICES_NUM}"
@@ -139,9 +139,9 @@ Describe 'service-parallel'
     test_parallel_GANTRY_MANIFEST_NUM_WORKERS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_NUM_WORKERS="NotANumber"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -183,9 +183,9 @@ Describe 'service-parallel'
     test_parallel_GANTRY_UPDATE_NUM_WORKERS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_NUM_WORKERS="NotANumber"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index 81288c3..ca2f9ea 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -28,10 +28,10 @@ Describe 'rollback'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Assume service update won't be done within TIMEOUT second.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -77,13 +77,13 @@ Describe 'rollback'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Assume service update won't be done within TIMEOUT second.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
       # Rollback would fail due to the incorrect option.
       # --with-registry-auth cannot be combined with --rollback.
       export GANTRY_ROLLBACK_OPTIONS="--with-registry-auth"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -129,11 +129,11 @@ Describe 'rollback'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Assume service update won't be done within TIMEOUT second.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
       export GANTRY_ROLLBACK_ON_FAILURE="false"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -179,7 +179,7 @@ Describe 'rollback'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Assume service update won't be done within TIMEOUT second.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
       # label should override the global environment variable.
@@ -188,7 +188,7 @@ Describe 'rollback'
       # --with-registry-auth cannot be combined with --rollback.
       local LABEL_AND_VALUE="gantry.rollback.options=--with-registry-auth"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -234,13 +234,13 @@ Describe 'rollback'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local TIMEOUT="${3}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Assume service update won't be done within TIMEOUT second.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
       # label should override the global environment variable.
       local LABEL_AND_VALUE="gantry.rollback.on_failure=false"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_timeout ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TIMEOUT}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_service_multiple_spec.sh b/tests/gantry_service_multiple_spec.sh
index 80ea6c3..955716c 100644
--- a/tests/gantry_service_multiple_spec.sh
+++ b/tests/gantry_service_multiple_spec.sh
@@ -45,11 +45,11 @@ Describe 'service-multiple-services'
       local SERVICE_NAME="${2}"
       local SERVICE_NAME1="${SERVICE_NAME}-1"
       local SERVICE_NAME2="${SERVICE_NAME}-2"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # test both the list of names and the filters
       export GANTRY_SERVICES_EXCLUDED="${SERVICE_NAME1}"
       export GANTRY_SERVICES_EXCLUDED_FILTERS="name=${SERVICE_NAME2}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     test_end() {
       local TEST_NAME="${1}"
diff --git a/tests/gantry_service_no_running_tasks_spec.sh b/tests/gantry_service_no_running_tasks_spec.sh
index 0ddd2ae..f080365 100644
--- a/tests/gantry_service_no_running_tasks_spec.sh
+++ b/tests/gantry_service_no_running_tasks_spec.sh
@@ -46,8 +46,8 @@ Describe "service-no-running-tasks"
     test_no_running_tasks_replicated() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     test_end() {
       local TEST_NAME="${1}"
@@ -120,8 +120,8 @@ Describe "service-no-running-tasks"
     test_no_running_tasks_global() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index 492ec7a..5e07452 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -26,8 +26,8 @@ Describe 'service-single-service'
     test_new_image_no() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -73,8 +73,8 @@ Describe 'service-single-service'
     test_new_image_yes() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -130,8 +130,8 @@ Describe 'service-single-service'
     test_new_image_no_digest() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -178,10 +178,10 @@ Describe 'service-single-service'
     test_new_image_SERVICES_SELF() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Explicitly set GANTRY_SERVICES_SELF
       export GANTRY_SERVICES_SELF="${SERVICE_NAME}"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index ea47e37..49b9495 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -36,10 +36,10 @@ Describe 'update-options'
       local LABEL_AND_VALUE=
       LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
       echo "Before updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
       echo "After updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
@@ -92,13 +92,13 @@ Describe 'update-options'
       local LABEL_AND_VALUE=
       LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
       echo "Before updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # label should override the global environment variable.
       export GANTRY_UPDATE_OPTIONS="--incorrect-option"
       local LABEL_AND_VALUE="gantry.update.options=--label-add=${LABEL}=${SERVICE_NAME}"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
       echo "After updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
@@ -147,9 +147,9 @@ Describe 'update-options'
     test_update_UPDATE_TIMEOUT_SECONDS_not_a_number() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -198,7 +198,7 @@ Describe 'update-options'
       local LABEL_AND_VALUE=
       LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
       echo "Before updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
-      reset_gantry_env "${SERVICE_NAME}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_UPDATE_OPTIONS="--label-add=${LABEL}=${SERVICE_NAME}"
       # label should override the global environment variable.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="NotANumber"
@@ -206,7 +206,7 @@ Describe 'update-options'
       LABEL_AND_VALUE="gantry.update.timeout_seconds=${TIMEOUT}"
       docker_service_update --label-add "${LABEL_AND_VALUE}" "${SERVICE_NAME}"
       local RETURN_VALUE=
-      run_gantry "${TEST_NAME}"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
       LABEL_AND_VALUE=$(_read_service_label "${SERVICE_NAME}" "${LABEL}")
       echo "After updating: LABEL_AND_VALUE=${LABEL_AND_VALUE}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 105ea23..c9c927e 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -69,9 +69,6 @@ export DONE_REMOVING_IMAGES="Done removing images"
 export SCHEDULE_NEXT_UPDATE_AT="Schedule next update at"
 export SLEEP_SECONDS_BEFORE_NEXT_UPDATE="Sleep [0-9]+ seconds before next update"
 
-export TEST_IMAGE_REMOVER="ghcr.io/shizunge/gantry-development"
-export TEST_SERVICE_IMAGE="alpine:latest"
-
 test_log() {
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^NONE$"  && return 0;
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^ERROR$"  && return 0;
@@ -199,8 +196,17 @@ _init_swarm() {
 
 # Image for the software under test (SUT)
 _get_sut_image() {
+  local SUITE_NAME="${1}"
   local SUT_REPO_TAG="${GANTRY_TEST_CONTAINER_REPO_TAG:-""}"
-  echo "${SUT_REPO_TAG}"
+  if [ -n "${SUT_REPO_TAG}" ]; then
+    echo "${SUT_REPO_TAG}"
+    return 0
+  fi
+  local TEST_CONTAINER="${GANTRY_TEST_CONTAINER:-""}"
+  if ! is_true "${TEST_CONTAINER}"; then
+    return 0
+  fi
+  _get_gantry_image "${SUITE_NAME}"
 }
 
 _get_initial_port() {
@@ -229,7 +235,7 @@ _next_available_port() {
 _get_docker_config_file() {
   local REGISTRY="${1:?}"
   REGISTRY=$(echo "${REGISTRY}" | tr ':' '-')
-  echo "/tmp/TEST_DOCKER_CONFIG-${REGISTRY}"
+  echo "/tmp/gantry-test-docker-config-${REGISTRY}"
 }
 
 _get_docker_config_argument() {
@@ -385,6 +391,35 @@ _stop_registry() {
   return 0
 }
 
+_get_test_service_image() {
+  grep alpine Dockerfile | sed -E "s/.*(alpine:)/\1/"
+}
+
+_get_gantry_image() {
+  local SUITE_NAME="${1:?}"
+  echo "$(load_test_registry "${SUITE_NAME}")/gantry/iut:${SUITE_NAME}"
+}
+
+_build_and_push_gantry_image() {
+  local SUITE_NAME="${1:?}"
+  local IMAGE=
+  IMAGE="$(_get_gantry_image "${SUITE_NAME}")" || return 1
+  pull_image_if_not_exist "$(_get_test_service_image)"
+  echo "Building gantry image ${IMAGE}"
+  timeout 120 docker build --quiet -t "${IMAGE}" .
+  echo "Pushing gantry image ${IMAGE}"
+  # SC2046 (warning): Quote this to prevent word splitting.
+  # shellcheck disable=SC2046
+  docker $(_get_docker_config_argument "${IMAGE}") push --quiet "${IMAGE}"
+}
+
+_remove_gantry_image() {
+  local SUITE_NAME="${1:?}"
+  IMAGE="$(_get_gantry_image "${SUITE_NAME}")" || return 1
+  echo "Removing gantry image ${IMAGE}"
+  docker image rm "${IMAGE}"
+}
+
 initialize_all_tests() {
   local SUITE_NAME="${1:-"gantry"}"
   local ENFORCE_LOGIN="${2}"
@@ -397,7 +432,7 @@ initialize_all_tests() {
   echo "=============================="
   _init_swarm
   _start_registry "${SUITE_NAME}" "${ENFORCE_LOGIN}"
-  pull_image_if_not_exist "${TEST_IMAGE_REMOVER}"
+  _build_and_push_gantry_image "${SUITE_NAME}"
 }
 
 # finish_all_tests should return non zero when there are errors.
@@ -405,6 +440,7 @@ finish_all_tests() {
   local SUITE_NAME="${1:-"gantry"}"
   local ENFORCE_LOGIN="${2}"
   SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-')
+  _remove_gantry_image "${SUITE_NAME}"
   _stop_registry "${SUITE_NAME}" "${ENFORCE_LOGIN}"
   echo "=============================="
   echo "== Finished all tests in ${SUITE_NAME}"
@@ -419,7 +455,8 @@ initialize_test() {
 }
 
 reset_gantry_env() {
-  local SERVICE_NAME="${1}"
+  local SUITE_NAME="${1}"
+  local SERVICE_NAME="${2}"
   export GANTRY_TEST_HOST_TO_CONTAINER=
   export GANTRY_TEST_DOCKER_CONFIG=
   export GANTRY_TEST_DOCKER_HOST=
@@ -455,7 +492,8 @@ reset_gantry_env() {
   export GANTRY_UPDATE_TIMEOUT_SECONDS=
   export GANTRY_CLEANUP_IMAGES=
   export GANTRY_CLEANUP_IMAGES_OPTIONS=
-  export GANTRY_CLEANUP_IMAGES_REMOVER="${TEST_IMAGE_REMOVER}"
+  export GANTRY_CLEANUP_IMAGES_REMOVER=
+  GANTRY_CLEANUP_IMAGES_REMOVER=$(_get_gantry_image "${SUITE_NAME}")
   export GANTRY_IMAGES_TO_REMOVE=
   export GANTRY_NOTIFICATION_APPRISE_URL=
   export GANTRY_NOTIFICATION_CONDITION=
@@ -572,9 +610,9 @@ build_test_image() {
   fi
   local FILE=
   FILE=$(mktemp)
-  echo "FROM ${TEST_SERVICE_IMAGE}" > "${FILE}"
+  echo "FROM $(_get_test_service_image)" > "${FILE}"
   echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
-  pull_image_if_not_exist "${TEST_SERVICE_IMAGE}"
+  pull_image_if_not_exist "$(_get_test_service_image)"
   echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
   timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
   rm "${FILE}"
@@ -659,9 +697,10 @@ _location_constraints() {
 
 pull_image_if_not_exist() {
   local IMAGE="${1}"
-  if ! docker image inspect "${IMAGE}" > /dev/null 2>&1; then
-    docker pull "${IMAGE}" > /dev/null
+  if docker image inspect "${IMAGE}" > /dev/null 2>&1; then
+    return 0
   fi
+  docker pull "${IMAGE}" > /dev/null
 }
 
 _enforce_login_enabled() {
@@ -883,17 +922,21 @@ _add_file_to_mount_options() {
   echo "${MOUNT_OPTIONS}"
 }
 
+_get_gantry_sut_name() {
+  local STACK="${1}"
+  local SERVICE_NAME=
+  SERVICE_NAME="gantry-test-SUT-${STACK}"
+  _sanitize_test_service_name "${SERVICE_NAME}"
+}
+
 stop_gantry_container() {
   local STACK="${1}"
-  local SUT_REPO_TAG=
-  SUT_REPO_TAG="$(_get_sut_image)"
-  if [ -z "${SUT_REPO_TAG}" ]; then
-    return 0;
+  local SERVICE_NAME=
+  SERVICE_NAME=$(_get_gantry_sut_name "${STACK}")
+  if ! docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" >/dev/null 2>&1; then
+    return 0
   fi
   local RETURN_VALUE=0
-  local SERVICE_NAME=
-  SERVICE_NAME="gantry-test-SUT-${STACK}"
-  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
   local CMD_OUTPUT=
   docker service logs --raw "${SERVICE_NAME}"
   if ! CMD_OUTPUT=$(docker service rm "${SERVICE_NAME}" 2>&1); then
@@ -908,8 +951,7 @@ _run_gantry_container() {
   local SUT_REPO_TAG="${2}"
   pull_image_if_not_exist "${SUT_REPO_TAG}"
   local SERVICE_NAME=
-  SERVICE_NAME="gantry-test-SUT-${STACK}"
-  SERVICE_NAME=$(_sanitize_test_service_name "${SERVICE_NAME}")
+  SERVICE_NAME=$(_get_gantry_sut_name "${STACK}")
   docker service rm "${SERVICE_NAME}" >/dev/null 2>&1;
   local MOUNT_OPTIONS=
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_TEST_HOST_TO_CONTAINER}")
@@ -991,7 +1033,8 @@ _restore_env() {
 }
 
 run_gantry() {
-  local STACK="${1}"
+  local SUITE_NAME="${1}"
+  local STACK="${2}"
   local DOCKER_CONFIG_SET=0
   local OLD_DOCKER_CONFIG=
   local DOCKER_HOST_SET=0
@@ -1006,7 +1049,7 @@ run_gantry() {
   fi
   local RETURN_VALUE=1
   local SUT_REPO_TAG=
-  SUT_REPO_TAG="$(_get_sut_image)"
+  SUT_REPO_TAG="$(_get_sut_image "${SUITE_NAME}")"
   if [ -n "${SUT_REPO_TAG}" ]; then
     _run_gantry_container "${STACK}" "${SUT_REPO_TAG}"
     RETURN_VALUE=$?

From ab3bc9c5a7e85cc675e13271cc8a43de5c6bfd6e Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Wed, 27 Nov 2024 21:02:08 -0800
Subject: [PATCH 25/27] [gantry] return non-zero when pre-run post-run failed.
 skip updating when pre-run failed.

---
 README.md                           | 2 +-
 src/entrypoint.sh                   | 2 ++
 tests/gantry_common_options_spec.sh | 3 ++-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4990fd0..1153267 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ You can configure the most behaviors of *Gantry* via environment variables.
 | GANTRY_LOG_LEVEL      | INFO | Control how many logs generated by *Gantry*. Valid values are `NONE`, `ERROR`, `WARN`, `INFO`, `DEBUG`. |
 | GANTRY_NODE_NAME      |      | Add node name to logs. If not set, *Gantry* will use the host name of the Docker Swarm's manager, which is read from either the Docker daemon socket of current node or `DOCKER_HOST`. |
 | GANTRY_POST_RUN_CMD   |      | Command(s) to `eval` after each updating iteration. For [example](examples/prune-and-watchtower), you can use this to remove unused containers, networks and images and update standalone docker containers. |
-| GANTRY_PRE_RUN_CMD    |      | Command(s) to `eval` before each updating iteration. For [example](examples/prune-and-watchtower), you can use this to remove unused containers, networks and images and update standalone docker containers. |
+| GANTRY_PRE_RUN_CMD    |      | Command(s) to `eval` before each updating iteration. For [example](examples/prune-and-watchtower), you can use this to remove unused containers, networks and images and update standalone docker containers. If you changed *Gantry* configurations in the pre-run command(s), the new value would apply to the following updating. If the last pre-run command failed, *Gantry* would skip updating services. |
 | GANTRY_SLEEP_SECONDS  | 0    | Interval between two updates. Set it to 0 to run *Gantry* once and then exit. When this is a non-zero value, after an updating, *Gantry* will sleep until the next scheduled update. The actual sleep time is this value minus time spent on updating services. |
 | TZ                    |      | Set timezone for time in logs. |
 
diff --git a/src/entrypoint.sh b/src/entrypoint.sh
index e97caff..1041ba1 100755
--- a/src/entrypoint.sh
+++ b/src/entrypoint.sh
@@ -121,6 +121,7 @@ gantry() {
   local ACCUMULATED_ERRORS=0
 
   eval_cmd "pre-run" "${PRE_RUN_CMD}"
+  ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))
 
   log INFO "Starting."
   gantry_initialize "${STACK}"
@@ -152,6 +153,7 @@ gantry() {
   ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))
 
   eval_cmd "post-run" "${POST_RUN_CMD}"
+  ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))
 
   local TIME_ELAPSED=
   TIME_ELAPSED=$(time_elapsed_since "${START_TIME}")
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index 95680cb..e6f8424 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -155,7 +155,8 @@ Describe 'common-options'
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
       When run test_common_PRE_POST_RUN_CMD "${TEST_NAME}" "${SERVICE_NAME}"
-      The status should be success
+      # Updating should be successful, but post-run comamnd failed.
+      The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output

From a08caa0bdf667c2e212a3c09936bae8133631662 Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Fri, 29 Nov 2024 12:28:49 -0800
Subject: [PATCH 26/27] [gantry][tests] Handler warning better from docker
 command. Do not use /tmp folder for tests.

Sometimes there could be a problem read files from /tmp.
---
 .gitignore                               |   1 +
 README.md                                |   6 +-
 src/docker_hub_rate.sh                   |   6 +-
 src/entrypoint.sh                        |  20 +--
 src/lib-common.sh                        |  91 ++++++++-----
 src/lib-gantry.sh                        | 162 ++++++++++++-----------
 tests/gantry_common_options_spec.sh      |   7 +-
 tests/gantry_login_docker_config_spec.sh |  32 ++---
 tests/gantry_login_negative_spec.sh      |  71 +++++-----
 tests/gantry_login_spec.sh               |  34 ++---
 tests/gantry_notify_spec.sh              |  10 +-
 tests/gantry_rollback_spec.sh            |   7 +-
 tests/gantry_service_single_spec.sh      |   4 -
 tests/gantry_update_options_spec.sh      |  56 +++++++-
 tests/spec_gantry_test_helper.sh         | 136 ++++++++++++-------
 15 files changed, 369 insertions(+), 274 deletions(-)

diff --git a/.gitignore b/.gitignore
index 348c7ec..f08ef08 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 .shellspec-quick.log
 coverage
+gantry-test-tmp
\ No newline at end of file
diff --git a/README.md b/README.md
index 1153267..5e918a4 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,8 @@ You can configure the most behaviors of *Gantry* via environment variables.
 | GANTRY_SERVICES_EXCLUDED_FILTERS | `label=gantry.services.excluded=true` | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter), e.g. `label=project=project-a`. Exclude services which match the given filters from updating. The default value allows you to add label `gantry.services.excluded=true` to services to exclude them from updating. Note that multiple filters will be logical **ANDED**. |
 | GANTRY_SERVICES_FILTERS          | | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter) that are accepted by `docker service ls --filter` to select services to update, e.g. `label=project=project-a`. Note that multiple filters will be logical **ANDED**. Also see [How to filters multiple services by name](docs/faq.md#how-to-filters-multiple-services-by-name). |
 
+> NOTE: *Gantry* reads labels on the services not on the containers. The labels need to go to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section, if you are using docker compose files to setup your services.
+
 ### To check if new images are available
 
 | Environment Variable  | Default | Description |
@@ -105,6 +107,8 @@ You can configure the most behaviors of *Gantry* via environment variables.
 
 Labels can be added to services to modify the behavior of *Gantry* for particular services. When *Gantry* sees the following labels on a service, it will modify the Docker command line only for that service. The value on the label overrides the global environment variables.
 
+> NOTE: *Gantry* reads labels on the services not on the containers. The labels need to go to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section, if you are using docker compose files to setup your services.
+
 | Label  | Description |
 |--------|-------------|
 | `gantry.auth.config=<configuration>`     | Override [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables). See [Authentication](docs/authentication.md). |
@@ -117,8 +121,6 @@ Labels can be added to services to modify the behavior of *Gantry* for particula
 | `gantry.update.options=<string>`         | Override [`GANTRY_UPDATE_OPTIONS`](#to-add-options-to-services-update). |
 | `gantry.update.timeout_seconds=<number>` | Override [`GANTRY_UPDATE_TIMEOUT_SECONDS`](#to-add-options-to-services-update). |
 
-> NOTE: You must apply the labels to the services not the containers. If you are using docker compose files to setup your services, you need to add the label to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section.
-
 ## FAQ
 
 [Authentication](docs/authentication.md)
diff --git a/src/docker_hub_rate.sh b/src/docker_hub_rate.sh
index 0d46fe7..be4e750 100755
--- a/src/docker_hub_rate.sh
+++ b/src/docker_hub_rate.sh
@@ -16,7 +16,7 @@
 #
 
 _curl_installed() {
-  curl --version 1>/dev/null 2>&1;
+  curl --version 1>/dev/null 2>/dev/null;
 }
 
 _docker_hub_rate_token() {
@@ -66,10 +66,10 @@ _docker_hub_echo_error() {
 docker_hub_rate() {
   local IMAGE="${1:-ratelimitpreview/test}"
   local USER_AND_PASS="${2}"
-  if ! type log 1>/dev/null 2>&1; then
+  if ! type log 1>/dev/null 2>/dev/null; then
     log() { echo "${*}" >&2; }
   fi
-  if ! type log_lines 1>/dev/null 2>&1; then
+  if ! type log_lines 1>/dev/null 2>/dev/null; then
     # Usage: echo "${LOGS}" | log_lines LEVLE
     log_lines() { local LEVEL="${1}"; while read -r LINE; do [ -z "${LINE}" ] && continue; log "${LEVEL}" "${LINE}"; done; }
   fi
diff --git a/src/entrypoint.sh b/src/entrypoint.sh
index 1041ba1..57281ae 100755
--- a/src/entrypoint.sh
+++ b/src/entrypoint.sh
@@ -63,7 +63,7 @@ load_libraries() {
 
 _run_on_node() {
   local HOST_NAME=
-  if ! HOST_NAME=$(docker node inspect self --format "{{.Description.Hostname}}" 2>&1); then
+  if ! HOST_NAME=$(run_cmd docker node inspect self --format "{{.Description.Hostname}}"); then
     log DEBUG "Failed to run \"docker node inspect self\": ${HOST_NAME}"
     return 1
   fi
@@ -73,18 +73,9 @@ _run_on_node() {
 
 _read_docker_hub_rate() {
   local HOST PASSWORD USER
-  if ! PASSWORD=$(gantry_read_registry_password 2>&1); then
-    log ERROR "Failed to read registry PASSWORD: ${PASSWORD}";
-    PASSWORD=
-  fi
-  if ! USER=$(gantry_read_registry_username 2>&1); then
-    log ERROR "Failed to read registry USER: ${USER}";
-    USER=
-  fi
-  if ! HOST=$(gantry_read_registry_host 2>&1); then
-    log ERROR "Failed to read registry HOST: ${HOST}";
-    HOST=
-  fi
+  USER=$(gantry_read_config "GANTRY_REGISTRY_USER")
+  PASSWORD=$(gantry_read_config "GANTRY_REGISTRY_PASSWORD")
+  HOST=$(gantry_read_config "GANTRY_REGISTRY_HOST")
   local USER_AND_PASS=
   if [ -n "${USER}" ] && [ -n "${PASSWORD}" ]; then
     if [ -z "${HOST}" ] || [ "${HOST}" = "docker.io" ]; then
@@ -107,6 +98,7 @@ gantry() {
   START_TIME=$(date +%s)
 
   [ -n "${DOCKER_HOST}" ] && log DEBUG "DOCKER_HOST=${DOCKER_HOST}"
+  [ -n "${DOCKER_CONFIG}" ] && log DEBUG "DOCKER_CONFIG=${DOCKER_CONFIG}"
   local RUN_ON_NODE=
   if ! RUN_ON_NODE=$(_run_on_node); then
     local HOST_STRING="${DOCKER_HOST:-"the current node"}"
@@ -123,7 +115,7 @@ gantry() {
   eval_cmd "pre-run" "${PRE_RUN_CMD}"
   ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))
 
-  log INFO "Starting."
+  log INFO "Starting Gantry."
   gantry_initialize "${STACK}"
   ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?))
 
diff --git a/src/lib-common.sh b/src/lib-common.sh
index 5b5ada2..27ef46d 100755
--- a/src/lib-common.sh
+++ b/src/lib-common.sh
@@ -39,8 +39,8 @@ grep_q() {
   # "grep -q" will exit immediately when the first line of data matches, and leading to broken pipe errors.
   grep -q -- "${@}";
   local GREP_RETURN=$?;
-  # Add "cat > /dev/null" to avoid broken pipe errors.
-  cat >/dev/null;
+  # Add "cat 1>/dev/null" to avoid broken pipe errors.
+  cat 1>/dev/null;
   return "${GREP_RETURN}"
 }
 
@@ -49,7 +49,7 @@ grep_q() {
 grep_q_i() {
   grep -q -i -- "${@}";
   local GREP_RETURN=$?;
-  cat >/dev/null;
+  cat 1>/dev/null;
   return "${GREP_RETURN}"
 }
 
@@ -242,7 +242,7 @@ _log_docker_time() {
   # date -d "${TIME_INPUT}" +"$(_time_format)" 2>/dev/null && return 0
   local EPOCH=
   if EPOCH=$(busybox date -d "${TIME_INPUT}" -D "%Y-%m-%dT%H:%M:%S" -u +%s 2>/dev/null); then
-    date -d "@${EPOCH}" +"$(_time_format)" 2>&1
+    date -d "@${EPOCH}" +"$(_time_format)"
     return 0
   fi
   if [ -n "${TIME_INPUT}" ]; then
@@ -367,7 +367,7 @@ read_config() {
     cat "${CONFIG_FILE}"
     return $?
   elif [ -n "${CONFIG_FILE}" ]; then
-    echo "Failed to read ${CONFIG_FILE}" >&2
+    echo "Failed to read file ${CONFIG_FILE}" >&2
     return 1
   fi
   eval "local CONFIG=\${${CONFIG_NAME}}"
@@ -434,13 +434,36 @@ eval_cmd() {
   return "${RETURN_VALUE}"
 }
 
+# When the command returns 0:
+# Echo stdout and log stderr as a warning. Return 0.
+# When the command returns non-zero:
+# Echo stdout + stderr. Return the same value from the docker command.
+run_cmd() {
+  local STDERR_STR=
+  local RETURN_VALUE=
+  # Use "3>&2 2>&1 1>&3" to swap stdout and stderr
+  { STDERR_STR=$("${@}" 3>&2 2>&1 1>&3); } 2>&1
+  RETURN_VALUE=$?
+
+  if [ -n "${STDERR_STR}" ]; then
+    if [ "${RETURN_VALUE}" = 0 ]; then
+      log WARN "${STDERR_STR} (From command: ${*})"
+    else
+      echo "${STDERR_STR}"
+    fi
+  fi
+  return "${RETURN_VALUE}"
+}
+
 swarm_network_arguments() {
   if [ -z "${NETWORK_NAME}" ]; then
     echo ""
     return 0
   fi
-  NETWORK_NAME=$(docker network ls --filter "name=${NETWORK_NAME}" --format '{{.Name}}')
-  if [ -z "${NETWORK_NAME}" ]; then
+  local RETURN_VALUE=
+  NETWORK_NAME=$(run_cmd docker network ls --filter "name=${NETWORK_NAME}" --format '{{.Name}}')
+  RETURN_VALUE=$?
+  if [ "${RETURN_VALUE}" != "0" ] || [ -z "${NETWORK_NAME}" ]; then
     echo ""
     return 0
   fi
@@ -481,7 +504,7 @@ docker_service_logs() {
 
 _docker_service_exists() {
   local SERVICE_NAME="${1}"
-  docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" >/dev/null 2>&1
+  docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" 1>/dev/null 2>/dev/null
 }
 
 _docker_wait_until_service_removed() {
@@ -495,7 +518,7 @@ _docker_wait_until_service_removed() {
 # This function will check the status of the service and stop the "docker service logs" command.
 _docker_service_logs_follow_and_stop() {
   local SERVICE_NAME="${1}"
-  ! _docker_service_exists "${SERVICE_NAME}" && return 1;
+  _docker_service_exists "${SERVICE_NAME}" || return 1;
   local PID=
   docker service logs --timestamps --no-task-ids --follow "${SERVICE_NAME}" 2>&1 &
   PID="${!}"
@@ -513,8 +536,8 @@ _docker_service_task_states() {
   local SERVICE_NAME="${1}"
   # We won't get the return value of the command via $? if we use "local STATES=$(command)".
   local STATES=
-  if ! STATES=$(docker service ps --no-trunc --format '[{{.Name}}][{{.Node}}] {{.CurrentState}} {{.Error}}' "${SERVICE_NAME}" 2>&1); then
-    echo "${STATES}" >&2
+  if ! STATES=$(run_cmd docker service ps --no-trunc --format '[{{.Name}}][{{.Node}}] {{.CurrentState}} {{.Error}}' "${SERVICE_NAME}"); then
+    log ERROR "${STATES}"
     return 1
   fi
   local NAME_LIST=
@@ -591,7 +614,7 @@ wait_service_state() {
   local RETURN_VALUE=0
   local DOCKER_CMD_ERROR=1
   local STATES=
-  while STATES=$(_docker_service_task_states "${SERVICE_NAME}" 2>&1); do
+  while STATES=$(_docker_service_task_states "${SERVICE_NAME}"); do
     DOCKER_CMD_ERROR=0
     RETURN_VALUE=$(_all_tasks_reach_state "${WANT_STATE}" "${CHECK_FAILURES}" "${STATES}") && break
     local SECONDS_ELAPSED=
@@ -606,7 +629,7 @@ wait_service_state() {
     DOCKER_CMD_ERROR=1
   done
   if [ "${DOCKER_CMD_ERROR}" != "0" ]; then
-    log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${STATES}"
+    log ERROR "Failed to obtain task states of service ${SERVICE_NAME}."
     return 1
   fi
   local LINE=
@@ -619,14 +642,12 @@ wait_service_state() {
 docker_service_remove() {
   local SERVICE_NAME="${1}"
   local POST_COMMAND="${2}"
-  ! _docker_service_exists "${SERVICE_NAME}" && return 0
+  _docker_service_exists "${SERVICE_NAME}" || return 0
   log DEBUG "Removing service ${SERVICE_NAME}."
-  local RETURN_VALUE=0
   local LOG=
-  if ! LOG=$(docker service rm "${SERVICE_NAME}" 2>&1); then
-    RETURN_VALUE=$?
+  if ! LOG=$(run_cmd docker service rm "${SERVICE_NAME}"); then
     log ERROR "Failed to remove docker service ${SERVICE_NAME}: ${LOG}"
-    return "${RETURN_VALUE}"
+    return 1
   fi
   if [ -n "${POST_COMMAND}" ]; then
     eval "${POST_COMMAND}"
@@ -654,7 +675,7 @@ docker_global_job() {
   SERVICE_NAME=$(_get_docker_command_name_arg "${@}")
   log INFO "Starting global-job ${SERVICE_NAME}."
   local LOG=
-  if ! LOG=$(docker service create --mode global-job "${@}"  2>&1); then
+  if ! LOG=$(run_cmd docker service create --mode global-job "${@}"); then
     log ERROR "Failed to create global-job ${SERVICE_NAME}: ${LOG}"
     return 1
   fi
@@ -669,7 +690,7 @@ docker_replicated_job() {
   # The Docker CLI does not exit on failures.
   log INFO "Starting replicated-job ${SERVICE_NAME}."
   local LOG=
-  if ! LOG=$(docker service create --mode replicated-job --detach "${@}" 2>&1); then
+  if ! LOG=$(run_cmd docker service create --mode replicated-job --detach "${@}"); then
     log ERROR "Failed to create replicated-job ${SERVICE_NAME}: ${LOG}"
     return 1
   fi
@@ -682,10 +703,10 @@ docker_replicated_job() {
 
 docker_version() {
   local cver capi sver sapi
-  if ! cver=$(docker version --format '{{.Client.Version}}' 2>&1);    then log ERROR "${cver}"; cver="error"; fi
-  if ! capi=$(docker version --format '{{.Client.APIVersion}}' 2>&1); then log ERROR "${capi}"; capi="error"; fi
-  if ! sver=$(docker version --format '{{.Server.Version}}' 2>&1);    then log ERROR "${sver}"; sver="error"; fi
-  if ! sapi=$(docker version --format '{{.Server.APIVersion}}' 2>&1); then log ERROR "${sapi}"; sapi="error"; fi
+  if ! cver=$(run_cmd docker version --format '{{.Client.Version}}');    then log ERROR "${cver}"; cver="error"; fi
+  if ! capi=$(run_cmd docker version --format '{{.Client.APIVersion}}'); then log ERROR "${capi}"; capi="error"; fi
+  if ! sver=$(run_cmd docker version --format '{{.Server.Version}}');    then log ERROR "${sver}"; sver="error"; fi
+  if ! sapi=$(run_cmd docker version --format '{{.Server.APIVersion}}'); then log ERROR "${sapi}"; sapi="error"; fi
   echo "Docker version client ${cver} (API ${capi}) server ${sver} (API ${sapi})"
 }
 
@@ -694,7 +715,7 @@ docker_version() {
 # return 1 when there is an error.
 docker_current_container_name() {
   local ALL_NETWORKS=
-  ALL_NETWORKS=$(docker network ls --format '{{.ID}}') || return 1;
+  ALL_NETWORKS=$(run_cmd docker network ls --format '{{.ID}}') || return 1;
   [ -z "${ALL_NETWORKS}" ] && return 0;
   local IPS=;
   # Get the string after "src":
@@ -702,8 +723,8 @@ docker_current_container_name() {
   IPS=$(ip route | grep src | sed -n -E "s/.* src (\S+).*$/\1/p");
   [ -z "${IPS}" ] && return 0;
   local GWBRIDGE_NETWORK HOST_NETWORK;
-  GWBRIDGE_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1;
-  HOST_NETWORK=$(docker network ls --format '{{.ID}}' --filter 'name=^host$') || return 1;
+  GWBRIDGE_NETWORK=$(run_cmd docker network ls --format '{{.ID}}' --filter 'name=^docker_gwbridge$') || return 1;
+  HOST_NETWORK=$(run_cmd docker network ls --format '{{.ID}}' --filter 'name=^host$') || return 1;
   local NID=;
   for NID in ${ALL_NETWORKS}; do
     # The output of gwbridge does not contain the container name. It looks like gateway_8f55496ce4f1/172.18.0.5/16.
@@ -711,7 +732,7 @@ docker_current_container_name() {
     # The output of host does not contain an IP.
     [ "${NID}" = "${HOST_NETWORK}" ] && continue;
     local ALL_LOCAL_NAME_AND_IP=;
-    ALL_LOCAL_NAME_AND_IP=$(docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1;
+    ALL_LOCAL_NAME_AND_IP=$(run_cmd docker network inspect "${NID}" --format "{{range .Containers}}{{.Name}}/{{println .IPv4Address}}{{end}}") || return 1;
     local NAME_AND_IP=;
     for NAME_AND_IP in ${ALL_LOCAL_NAME_AND_IP}; do
       [ -z "${NAME_AND_IP}" ] && continue;
@@ -747,10 +768,10 @@ docker_remove() {
   fi
   log DEBUG "Removing container ${CNAME}."
   if [ "${STATUS}" = "running" ]; then
-    docker stop "${CNAME}" >/dev/null 2>/dev/null
+    docker container stop "${CNAME}" 1>/dev/null 2>/dev/null
   fi
   # If the container is created with "--rm", it will be removed automatically when being stopped.
-  docker rm -f "${CNAME}" >/dev/null;
+  docker container rm -f "${CNAME}" 1>/dev/null;
   log INFO "Removed container ${CNAME}."
 }
 
@@ -758,15 +779,15 @@ docker_run() {
   local RETRIES=0
   local MAX_RETRIES=5
   local SLEEP_SECONDS=10
-  local MSG=
-  while ! MSG=$(docker run "${@}" 2>&1); do
+  local LOG=
+  while ! LOG=$(run_cmd docker container run "${@}"); do
     if [ ${RETRIES} -ge ${MAX_RETRIES} ]; then
-      log ERROR "Failed to run docker. Reached the max retries ${MAX_RETRIES}. ${MSG}"
+      log ERROR "Failed to run docker. Reached the max retries ${MAX_RETRIES}. ${LOG}"
       return 1
     fi
     RETRIES=$((RETRIES + 1))
     sleep ${SLEEP_SECONDS}
-    log WARN "Retry docker run (${RETRIES}). ${MSG}"
+    log WARN "Retry docker container run (${RETRIES}). ${LOG}"
   done
-  echo "${MSG}"
+  echo "${LOG}"
 }
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index 3f0a21a..f573b8f 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -16,12 +16,12 @@
 #
 
 # This function calls read_env() underneath.
-# read_env() returns an empty string if ENV_VALUE is set to empty,
-# in which case we want to use the DEFAULT_VALUE.
 _read_env_default() {
   local ENV_NAME="${1}"
   local DEFAULT_VALUE="${2}"
   local READ_VALUE=
+  # read_env() returns an empty string if ENV_VALUE is set, but is empty,
+  # in which case we want to use the DEFAULT_VALUE.
   READ_VALUE=$(read_env "${ENV_NAME}" "${DEFAULT_VALUE}")
   local VALUE="${READ_VALUE}"
   [ -z "${VALUE}" ] && VALUE="${DEFAULT_VALUE}"
@@ -33,7 +33,7 @@ _read_env_default() {
 gantry_read_number() {
   local ENV_NAME="${1}"
   local DEFAULT_VALUE="${2}"
-  ! is_number "${DEFAULT_VALUE}" && log ERROR "DEFAULT_VALUE must be a number. Got \"${DEFAULT_VALUE}\"." && return 1
+  ! is_number "${DEFAULT_VALUE}" && log ERROR "DEFAULT_VALUE for ${ENV_NAME} must be a number. Got \"${DEFAULT_VALUE}\"." && return 1
   local VALUE=
   VALUE=$(_read_env_default "${ENV_NAME}" "${DEFAULT_VALUE}")
   if ! is_number "${VALUE}"; then
@@ -49,7 +49,7 @@ _get_label_from_service() {
   local SERVICE_NAME="${1}"
   local LABEL="${2}"
   local VALUE=
-  if ! VALUE=$(docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}" 2>&1); then
+  if ! VALUE=$(run_cmd docker service inspect -f "{{index .Spec.Labels \"${LABEL}\"}}" "${SERVICE_NAME}"); then
     log ERROR "Failed to obtain the value of label ${LABEL} from service ${SERVICE_NAME}. ${VALUE}"
     return 1
   fi
@@ -134,7 +134,7 @@ _login_registry() {
   local LOGIN_MSG=
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! LOGIN_MSG=$(echo "${PASSWORD}" | docker ${AUTH_CONFIG} login --username="${USER}" --password-stdin "${HOST}" 2>&1); then
+  if ! LOGIN_MSG=$(echo "${PASSWORD}" | run_cmd docker ${AUTH_CONFIG} login --username="${USER}" --password-stdin "${HOST}"); then
     log ERROR "Failed to login to ${REGISTRY_CONFIG_MESSAGE}. ${LOGIN_MSG}"
     return 1
   fi
@@ -146,42 +146,24 @@ _login_registry() {
   return 0
 }
 
-gantry_read_registry_username() {
-  read_config GANTRY_REGISTRY_USER
-}
-
-gantry_read_registry_password() {
-  read_config GANTRY_REGISTRY_PASSWORD
-}
-
-gantry_read_registry_host() {
-  read_config GANTRY_REGISTRY_HOST
+gantry_read_config() {
+  local CONFIG_NAME="${1}"
+  local CONFIG_VALUE=
+  if ! CONFIG_VALUE=$(read_config "${CONFIG_NAME}" 2>&1); then
+    log ERROR "Failed to read ${CONFIG_NAME}: ${CONFIG_VALUE}"
+    return 1
+  fi
+  echo "${CONFIG_VALUE}"
 }
 
 _authenticate_to_registries() {
   local CONFIGS_FILE="${GANTRY_REGISTRY_CONFIGS_FILE:-""}"
   local ACCUMULATED_ERRORS=0
   local CONFIG HOST PASSWORD USER
-  if ! CONFIG=$(read_config GANTRY_REGISTRY_CONFIG 2>&1); then
-    log ERROR "Failed to read registry CONFIG: ${CONFIG}"
-    ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
-    CONFIG=
-  fi
-  if ! HOST=$(gantry_read_registry_host 2>&1); then
-    log ERROR "Failed to read registry HOST: ${HOST}"
-    ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
-    HOST=
-  fi
-  if ! PASSWORD=$(gantry_read_registry_password 2>&1); then
-    log ERROR "Failed to read registry PASSWORD: ${PASSWORD}"
-    ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
-    PASSWORD=
-  fi
-  if ! USER=$(gantry_read_registry_username 2>&1); then
-    log ERROR "Failed to read registry USER: ${USER}"
-    ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
-    USER=
-  fi
+  CONFIG=$(gantry_read_config "GANTRY_REGISTRY_CONFIG") || ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
+  HOST=$(gantry_read_config "GANTRY_REGISTRY_HOST") || ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
+  PASSWORD=$(gantry_read_config "GANTRY_REGISTRY_PASSWORD") || ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
+  USER=$(gantry_read_config "GANTRY_REGISTRY_USER") || ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + 1))
   if [ "${ACCUMULATED_ERRORS}" -gt 0 ]; then
     log ERROR "Skip logging in due to previous error(s)."
   else
@@ -229,9 +211,7 @@ _send_notification() {
   local TYPE="${1}"
   local TITLE="${2}"
   local BODY="${3}"
-  if ! type notify_summary >/dev/null 2>&1; then
-    return 0
-  fi
+  type notify_summary 1>/dev/null 2>/dev/null || return 0
   notify_summary "${TYPE}" "${TITLE}" "${BODY}"
 }
 
@@ -266,9 +246,9 @@ _get_static_variables_folder() {
 }
 
 _remove_static_variables_folder() {
-  [ -z "${STATIC_VARIABLES_FOLDER}" ] && return 0
   local TO_REMOVE_STATIC_VARIABLES_FOLDER=
   TO_REMOVE_STATIC_VARIABLES_FOLDER="$(_get_static_variables_folder_name)"
+  [ ! -d "${TO_REMOVE_STATIC_VARIABLES_FOLDER}" ] && return 0
   log DEBUG "Removing STATIC_VARIABLES_FOLDER ${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
   unset STATIC_VARIABLES_FOLDER
   rm -r "${TO_REMOVE_STATIC_VARIABLES_FOLDER}"
@@ -286,14 +266,14 @@ _lock() {
   local NAME="${1}"
   local LOCK_NAME=
   LOCK_NAME="$(_get_static_variables_folder)/${NAME}-LOCK"
-  while ! mkdir "${LOCK_NAME}" >/dev/null 2>&1; do sleep 0.001; done
+  while ! mkdir "${LOCK_NAME}" 1>/dev/null 2>/dev/null; do sleep 0.001; done
 }
 
 _unlock() {
   local NAME="${1}"
   local LOCK_NAME=
   LOCK_NAME="$(_get_static_variables_folder)/${NAME}-LOCK"
-  rm -r "${LOCK_NAME}" >/dev/null 2>&1
+  rm -r "${LOCK_NAME}" 1>/dev/null 2>/dev/null
 }
 
 _static_variable_read_list_core() {
@@ -364,15 +344,15 @@ _remove_container() {
   local IMAGE="${1}";
   local STATUS="${2}";
   local CIDS=
-  if ! CIDS=$(docker container ls --all --filter "ancestor=${IMAGE}" --filter "status=${STATUS}" --format '{{.ID}}' 2>&1); then
+  if ! CIDS=$(run_cmd docker container ls --all --filter "ancestor=${IMAGE}" --filter "status=${STATUS}" --format '{{.ID}}'); then
     log ERROR "Failed to list ${STATUS} containers with image ${IMAGE}.";
     echo "${CIDS}" | log_lines ERROR
     return 1;
   fi
   local CID CNAME CRM_MSG
   for CID in ${CIDS}; do
-    CNAME=$(docker container inspect --format '{{.Name}}' "${CID}");
-    if ! CRM_MSG=$(docker container rm "${CID}" 2>&1); then
+    CNAME=$(run_cmd docker container inspect --format '{{.Name}}' "${CID}");
+    if ! CRM_MSG=$(run_cmd docker container rm "${CID}"); then
       log ERROR "Failed to remove ${STATUS} container ${CNAME}, which is using image ${IMAGE}.";
       echo "${CRM_MSG}" | log_lines ERROR
       continue;
@@ -387,13 +367,13 @@ gantry_remove_images() {
   log DEBUG "$(docker_version)"
   local IMAGE=
   for IMAGE in ${IMAGES_TO_REMOVE}; do
-    if ! docker image inspect "${IMAGE}" 1>/dev/null 2>&1 ; then
+    if ! run_cmd docker image inspect "${IMAGE}" 1>/dev/null; then
       log DEBUG "There is no image ${IMAGE} on the node.";
       continue;
     fi
     _remove_container "${IMAGE}" exited;
     _remove_container "${IMAGE}" dead;
-    if ! RMI_MSG=$(docker rmi "${IMAGE}" 2>&1); then
+    if ! RMI_MSG=$(run_cmd docker image rm "${IMAGE}"); then
       log ERROR "Failed to remove image ${IMAGE}.";
       echo "${RMI_MSG}" | log_lines ERROR
       continue;
@@ -446,6 +426,7 @@ _remove_images() {
   # shellcheck disable=SC2086
   docker_global_job --name "${SERVICE_NAME}" \
     --detach=true \
+    --with-registry-auth \
     --restart-condition on-failure \
     --restart-max-attempts 1 \
     --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock \
@@ -616,7 +597,9 @@ gantry_current_service_name() {
   CNAME=$(_current_container_name) || return 1
   [ -z "${CNAME}" ] && return 0
   local SNAME=
-  SNAME=$(docker container inspect "${CNAME}" --format '{{range $key,$value := .Config.Labels}}{{$key}}={{println $value}}{{end}}' \
+  # SC2016 (info): Expressions don't expand in single quotes, use double quotes for that.
+  # shellcheck disable=SC2016
+  SNAME=$(run_cmd docker container inspect "${CNAME}" --format '{{range $key,$value := .Config.Labels}}{{$key}}={{println $value}}{{end}}' \
     | grep "com.docker.swarm.service.name" \
     | sed -n -E "s/com.docker.swarm.service.name=(.*)$/\1/p") || return 1
   _static_variable_add_unique_to_list STATIC_VAR_CURRENT_SERVICE_NAME "${SNAME}"
@@ -643,19 +626,37 @@ _service_is_self() {
 _get_service_image() {
   local SERVICE_NAME="${1}"
   [ -z "${SERVICE_NAME}" ] && return 1
-  docker service inspect -f '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "${SERVICE_NAME}"
+  local RETURN_VALUE=
+  local IMAGE_WITH_DIGEST=
+  IMAGE_WITH_DIGEST=$(run_cmd docker service inspect -f '{{.Spec.TaskTemplate.ContainerSpec.Image}}' "${SERVICE_NAME}")
+  RETURN_VALUE=$?
+  if [ "${RETURN_VALUE}" != "0" ]; then
+    log ERROR "Failed to obtain image from service ${SERVICE_NAME}. ${IMAGE_WITH_DIGEST}"
+  else
+    echo "${IMAGE_WITH_DIGEST}"
+  fi
+  return "${RETURN_VALUE}"
 }
 
 _get_service_previous_image() {
   local SERVICE_NAME="${1}"
   [ -z "${SERVICE_NAME}" ] && return 1
-  docker service inspect -f '{{.PreviousSpec.TaskTemplate.ContainerSpec.Image}}' "${SERVICE_NAME}"
+  local RETURN_VALUE=
+  local IMAGE_WITH_DIGEST=
+  IMAGE_WITH_DIGEST=$(run_cmd docker service inspect -f '{{.PreviousSpec.TaskTemplate.ContainerSpec.Image}}' "${SERVICE_NAME}")
+  RETURN_VALUE=$?
+  if [ "${RETURN_VALUE}" != "0" ]; then
+    log ERROR "Failed to obtain previous image from service ${SERVICE_NAME}. ${IMAGE_WITH_DIGEST}"
+  else
+    echo "${IMAGE_WITH_DIGEST}"
+  fi
+  return "${RETURN_VALUE}"
 }
 
 _get_service_mode() {
   local SERVICE_NAME="${1}"
   local MODE=
-  if ! MODE=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Mode}} {{.Name}}' 2>&1); then
+  if ! MODE=$(run_cmd docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Mode}} {{.Name}}'); then
     log ERROR "Failed to obtain the mode of the service ${SERVICE_NAME}: ${MODE}"
     return 1
   fi
@@ -751,13 +752,13 @@ _get_image_info() {
     [ -n "${MANIFEST_OPTIONS}" ] && log INFO "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker buildx imagetools inspect\"."
     # SC2086: Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2086
-    MSG=$(docker ${AUTH_CONFIG} buildx imagetools inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
+    MSG=$(run_cmd docker ${AUTH_CONFIG} buildx imagetools inspect ${MANIFEST_OPTIONS} "${IMAGE}");
     RETURN_VALUE=$?
   elif echo "${MANIFEST_CMD}" | grep_q_i "manifest"; then
     [ -n "${MANIFEST_OPTIONS}" ] && log INFO "Adding options \"${MANIFEST_OPTIONS}\" to the command \"docker manifest inspect\"."
     # SC2086: Double quote to prevent globbing and word splitting.
     # shellcheck disable=SC2086
-    MSG=$(docker ${AUTH_CONFIG} manifest inspect ${MANIFEST_OPTIONS} "${IMAGE}" 2>&1);
+    MSG=$(run_cmd docker ${AUTH_CONFIG} manifest inspect ${MANIFEST_OPTIONS} "${IMAGE}");
     RETURN_VALUE=$?
   elif echo "${MANIFEST_CMD}" | grep_q_i "none"; then
     # We should never reach here, the "none" command is already checked inside the function _inspect_image.
@@ -782,10 +783,7 @@ _inspect_image() {
   local MANIFEST_CMD=
   MANIFEST_CMD=$(_read_env_or_label "${SERVICE_NAME}" "GANTRY_MANIFEST_CMD" "gantry.manifest.cmd" "buildx")
   local IMAGE_WITH_DIGEST=
-  if ! IMAGE_WITH_DIGEST=$(_get_service_image "${SERVICE_NAME}" 2>&1); then
-    log ERROR "Failed to obtain image from service ${SERVICE_NAME}. ${IMAGE_WITH_DIGEST}"
-    return 1
-  fi
+  IMAGE_WITH_DIGEST=$(_get_service_image "${SERVICE_NAME}") || return $?
   local IMAGE=
   local DIGEST=
   IMAGE=$(extract_string "${IMAGE_WITH_DIGEST}" '@' 1)
@@ -869,7 +867,7 @@ _inspect_service() {
 _get_number_of_running_tasks() {
   local SERVICE_NAME="${1}"
   local REPLICAS=
-  if ! REPLICAS=$(docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Replicas}} {{.Name}}' 2>&1); then
+  if ! REPLICAS=$(run_cmd docker service ls --filter "name=${SERVICE_NAME}" --format '{{.Replicas}} {{.Name}}'); then
     log ERROR "Failed to obtain task states of service ${SERVICE_NAME}: ${REPLICAS}"
     return 1
   fi
@@ -903,8 +901,8 @@ _get_service_update_additional_options() {
   local NUM_RUNS=
   NUM_RUNS=$(_get_number_of_running_tasks "${SERVICE_NAME}")
   ! is_number "${NUM_RUNS}" && log WARN "NUM_RUNS \"${NUM_RUNS}\" is not a number." && return 1
-  local OPTIONS=""
-  local SPACE=""
+  local OPTIONS=
+  local SPACE=
   if [ "${NUM_RUNS}" = "0" ]; then
     # Add "--detach=true" when there is no running tasks.
     # https://github.com/docker/cli/issues/627
@@ -950,18 +948,18 @@ _rollback_service() {
   # "service update --rollback" needs to take different options from "service update"
   local AUTOMATIC_OPTIONS=
   AUTOMATIC_OPTIONS=$(_get_service_rollback_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
-  [ -n "${AUTOMATIC_OPTIONS}" ] && log INFO "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
-  [ -n "${ROLLBACK_OPTIONS}" ] && log INFO "Adding options \"${ROLLBACK_OPTIONS}\" specified by user to the command \"docker service update --rollback\" for ${SERVICE_NAME}."
+  local CMD_STRING="\"docker service update --rollback\""
+  [ -n "${AUTH_CONFIG}" ] && log INFO "Adding options \"${AUTH_CONFIG}\" to the command ${CMD_STRING} for ${SERVICE_NAME}."
+  [ -n "${AUTOMATIC_OPTIONS}" ] && log INFO "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command ${CMD_STRING} for ${SERVICE_NAME}."
+  [ -n "${ROLLBACK_OPTIONS}" ] && log INFO "Adding options \"${ROLLBACK_OPTIONS}\" specified by user to the command ${CMD_STRING} for ${SERVICE_NAME}."
   local ROLLBACK_MSG=
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! ROLLBACK_MSG=$(docker ${AUTH_CONFIG} service update --quiet ${AUTOMATIC_OPTIONS} ${ROLLBACK_OPTIONS} --rollback "${SERVICE_NAME}" 2>&1); then
+  if ! ROLLBACK_MSG=$(run_cmd docker ${AUTH_CONFIG} service update --quiet ${AUTOMATIC_OPTIONS} ${ROLLBACK_OPTIONS} --rollback "${SERVICE_NAME}"); then
     log ERROR "Failed to roll back ${SERVICE_NAME}. ${ROLLBACK_MSG}"
     return 1
   fi
-  # Usually the ROLLBACK_MSG is same as the SERVICE_NAME.
-  [ "${ROLLBACK_MSG}" != "${SERVICE_NAME}" ] && log WARN "There are additional messages from rolling back ${SERVICE_NAME}: ${ROLLBACK_MSG}"
   log INFO "Rolled back ${SERVICE_NAME}."
 }
 
@@ -976,7 +974,7 @@ _get_timeout_command() {
     _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_INPUT_ERROR "${SERVICE_NAME}"
     return 1
   fi
-  local TIMEOUT_COMMAND=""
+  local TIMEOUT_COMMAND=
   if [ "${UPDATE_TIMEOUT_SECONDS}" != "0" ]; then
     TIMEOUT_COMMAND="timeout ${UPDATE_TIMEOUT_SECONDS}"
     log INFO "Set timeout to ${UPDATE_TIMEOUT_SECONDS} for updating ${SERVICE_NAME}."
@@ -1005,19 +1003,23 @@ _update_single_service() {
   local AUTOMATIC_OPTIONS=
   AUTH_CONFIG=$(_get_auth_config_from_service "${SERVICE_NAME}")
   AUTOMATIC_OPTIONS=$(_get_service_update_additional_options "${SERVICE_NAME}" "${AUTH_CONFIG}")
-  [ -n "${AUTH_CONFIG}" ] && log INFO "Adding options \"${AUTH_CONFIG}\" to docker commands for ${SERVICE_NAME}."
-  [ -n "${AUTOMATIC_OPTIONS}" ] && log INFO "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command \"docker service update\" for ${SERVICE_NAME}."
-  [ -n "${UPDATE_OPTIONS}" ] && log INFO "Adding options \"${UPDATE_OPTIONS}\" specified by user to the command \"docker service update\" for ${SERVICE_NAME}."
-  local TIMEOUT_COMMAND=""
+  local CMD_STRING="\"docker service update\""
+  [ -n "${AUTH_CONFIG}" ] && log INFO "Adding options \"${AUTH_CONFIG}\" to the command ${CMD_STRING} for ${SERVICE_NAME}."
+  [ -n "${AUTOMATIC_OPTIONS}" ] && log INFO "Adding options \"${AUTOMATIC_OPTIONS}\" automatically to the command ${CMD_STRING} for ${SERVICE_NAME}."
+  [ -n "${UPDATE_OPTIONS}" ] && log INFO "Adding options \"${UPDATE_OPTIONS}\" specified by user to the command ${CMD_STRING} for ${SERVICE_NAME}."
+  local TIMEOUT_COMMAND=
   TIMEOUT_COMMAND=$(_get_timeout_command "${SERVICE_NAME}") || return 1
-  local UPDATE_COMMAND="${TIMEOUT_COMMAND} docker ${AUTH_CONFIG} service update"
+  local SPACE_T=
+  [ -n "${TIMEOUT_COMMAND}" ] && SPACE_T=" "
+  local SPACE_C=
+  [ -n "${AUTH_CONFIG}" ] && SPACE_C=" "
+  local UPDATE_COMMAND="${TIMEOUT_COMMAND}${SPACE_T}docker ${AUTH_CONFIG}${SPACE_C}service update"
   local UPDATE_RETURN_VALUE=0
   local UPDATE_MSG=
-  # Add "2>/dev/null" outside the $(cmd) to suppress the "Terminated" message from "busybox timeout".
   # Add "-quiet" to suppress progress output.
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  UPDATE_MSG=$(${UPDATE_COMMAND} --quiet ${AUTOMATIC_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}" 2>&1) 2>/dev/null;
+  UPDATE_MSG=$(run_cmd ${UPDATE_COMMAND} --quiet ${AUTOMATIC_OPTIONS} ${UPDATE_OPTIONS} --image="${IMAGE}" "${SERVICE_NAME}");
   UPDATE_RETURN_VALUE=$?
   if [ "${UPDATE_RETURN_VALUE}" != 0 ]; then
     # When there is a timeout:
@@ -1025,7 +1027,7 @@ _update_single_service() {
     # * busybox timeout returns 143
     local TIMEOUT_RETURN_CODE=124
     timeout --help 2>&1 | grep_q_i "BusyBox" && TIMEOUT_RETURN_CODE=143
-    local TIMEOUT_MSG=""
+    local TIMEOUT_MSG=
     if [ -n "${TIMEOUT_COMMAND}" ] && [ "${UPDATE_RETURN_VALUE}" = "${TIMEOUT_RETURN_CODE}" ]; then
       TIMEOUT_MSG="The return value ${UPDATE_RETURN_VALUE} indicates the job timed out."
     fi
@@ -1035,8 +1037,6 @@ _update_single_service() {
     _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_FAILED "${SERVICE_NAME}"
     return 1
   fi
-  # Usually the UPDATE_MSG is same as the SERVICE_NAME.
-  [ "${UPDATE_MSG}" != "${SERVICE_NAME}" ] && log WARN "There are additional messages from updating ${SERVICE_NAME}: ${UPDATE_MSG}"
   local TIME_ELAPSED=
   TIME_ELAPSED=$(time_elapsed_since "${START_TIME}")
   local PREVIOUS_IMAGE=
@@ -1090,15 +1090,17 @@ _run_parallel() {
 
 _get_services_filted() {
   local SERVICES_FILTERS="${1}"
-  local SERVICES=
   local FILTERS=
+  local SPACE=
   local F=
   for F in ${SERVICES_FILTERS}; do
-    FILTERS="${FILTERS} --filter ${F}"
+    FILTERS="${FILTERS}${SPACE}--filter ${F}"
+    SPACE=" "
   done
+  local SERVICES=
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
-  if ! SERVICES=$(docker service ls --quiet ${FILTERS} --format '{{.Name}}' 2>&1); then
+  if ! SERVICES=$(run_cmd docker service ls --quiet ${FILTERS} --format '{{.Name}}'); then
     log ERROR "Failed to obtain services list with \"${FILTERS}\". ${SERVICES}"
     return 1
   fi
@@ -1204,8 +1206,8 @@ gantry_finalize() {
   local STACK="${1:-gantry}"
   local NUM_ERRORS="${2:-0}"
   local RETURN_VALUE=0
-  ! _remove_images "${STACK}-image-remover" && RETURN_VALUE=1
-  ! _report_services "${STACK}" "${NUM_ERRORS}" && RETURN_VALUE=1
+  _remove_images "${STACK}-image-remover" || RETURN_VALUE=1
+  _report_services "${STACK}" "${NUM_ERRORS}" || RETURN_VALUE=1
   _remove_static_variables_folder
   return "${RETURN_VALUE}"
 }
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
index e6f8424..10066bd 100644
--- a/tests/gantry_common_options_spec.sh
+++ b/tests/gantry_common_options_spec.sh
@@ -102,9 +102,9 @@ Describe 'common-options'
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local ENV_BEFORE_RUN=
-      ENV_BEFORE_RUN=$(mktemp)
+      ENV_BEFORE_RUN=$(make_test_temp_file)
       local ENV_AFTER_RUN=
-      ENV_AFTER_RUN=$(mktemp)
+      ENV_AFTER_RUN=$(make_test_temp_file)
 
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # There should be no warnings or errors. So it should work the same as LOG_LEVLE=NONE.
@@ -113,7 +113,8 @@ Describe 'common-options'
       run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       declare -p > "${ENV_AFTER_RUN}"
       # Allow the 3 mismatches LOG_LEVEL NODE_NAME LOG_SCOPE used in log() function.
-      # Allow the 2 mismatches LINENO _ for kcov coverage.
+      # Allow the 1 mismatch LINENO for kcov coverage.
+      # Allow the 1 mismatch _ for the previous command.
       for ALLOWED in LOG_LEVEL NODE_NAME LOG_SCOPE LINENO _; do
         sed -i "s/^declare .* ${ALLOWED}=.*//" "${ENV_BEFORE_RUN}"
         sed -i "s/^declare .* ${ALLOWED}=.*//" "${ENV_AFTER_RUN}"
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
index 0bd8ece..d39b27a 100644
--- a/tests/gantry_login_docker_config_spec.sh
+++ b/tests/gantry_login_docker_config_spec.sh
@@ -42,7 +42,7 @@ Describe 'login-docker-config'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_no_label() {
       local TEST_NAME="${1}"
@@ -52,8 +52,8 @@ Describe 'login-docker-config'
       local USERNAME="${5}"
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
@@ -91,7 +91,6 @@ Describe 'login-docker-config'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
       # Gantry adds --with-registry-auth, because DOCKER_CONFIG and the configuration name are same (default location).
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -116,7 +115,7 @@ Describe 'login-docker-config'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_docker_config_default_config() {
       local TEST_NAME="${1}"
@@ -126,8 +125,8 @@ Describe 'login-docker-config'
       local USERNAME="${5}"
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       # Do not set GANTRY_AUTH_CONFIG_LABEL on the service.
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_TEST_DOCKER_CONFIG="${CONFIG}"
@@ -138,7 +137,7 @@ Describe 'login-docker-config'
       local RETURN_VALUE=
       run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      docker logout "${REGISTRY}" > /dev/null
+      docker logout "${REGISTRY}" 1>/dev/null
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
@@ -165,7 +164,6 @@ Describe 'login-docker-config'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
       # Gantry adds --with-registry-auth for using the default configuration.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -193,7 +191,8 @@ Describe 'login-docker-config'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
+    INCORRECT_CONFIG=$(get_config_name "incorrect-")
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_start() {
       local TEST_NAME="${1}"
@@ -211,13 +210,13 @@ Describe 'login-docker-config'
       local REGISTRY="${4}"
       local USERNAME="${5}"
       local PASSWORD="${6}"
+      local INCORRECT_CONFIG="${7}"
       local SERVICE_NAME0="${SERVICE_NAME}-0"
       local SERVICE_NAME1="${SERVICE_NAME}-1"
       local SERVICE_NAME2="${SERVICE_NAME}-2"
-      local INCORRECT_CONFIG="incorrect-${CONFIG}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       local CONFIGS_FILE=
-      CONFIGS_FILE=$(mktemp)
+      CONFIGS_FILE=$(make_test_temp_file)
       echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" > "${CONFIGS_FILE}"
       # Set GANTRY_AUTH_CONFIG_LABEL on SERVICE_NAME1, but not on SERVICE_NAME0.
       # Inspection of SERVICE_NAME0 should fail (incorrect label overrides DOCKER_CONFIG).
@@ -250,7 +249,7 @@ Describe 'login-docker-config'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_docker_config_label_override "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" "${INCORRECT_CONFIG}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -258,11 +257,11 @@ Describe 'login-docker-config'
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "${INCORRECT_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME0}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${INCORRECT_CONFIG}.*${SERVICE_NAME0}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME1}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME2}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME0}.*"
@@ -281,9 +280,6 @@ Describe 'login-docker-config'
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME1}"
       # Gantry adds --with-registry-auth, because DOCKER_CONFIG and the configuration name are same (default location).
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME2}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME0}.*"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME1}.*"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME2}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME0}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME1}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME2}"
diff --git a/tests/gantry_login_negative_spec.sh b/tests/gantry_login_negative_spec.sh
index 0af2a88..3e6d59b 100644
--- a/tests/gantry_login_negative_spec.sh
+++ b/tests/gantry_login_negative_spec.sh
@@ -26,7 +26,7 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_no_login() {
       local TEST_NAME="${1}"
@@ -55,7 +55,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       # No --with-registry-auth, because no login.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -80,7 +79,7 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_incorrect_password() {
       local TEST_NAME="${1}"
@@ -91,8 +90,8 @@ Describe 'login-negative'
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       local INCORRECT_PASSWORD="${PASSWORD}-incorrect-password"
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${INCORRECT_PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${INCORRECT_PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
@@ -127,7 +126,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -149,7 +147,7 @@ Describe 'login-negative'
     TEST_NAME="test_login_read_only_file"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    AUTH_CONFIG=$(mktemp -d)
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_read_only_file() {
       local TEST_NAME="${1}"
@@ -163,8 +161,8 @@ Describe 'login-negative'
       # So do not run the test with a container/image.
       mkdir -p "${CONFIG}"
       chmod 444 "${CONFIG}"
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Use GANTRY_TEST_HOST_TO_CONTAINER to mount the file from host to the container.
@@ -178,7 +176,7 @@ Describe 'login-negative'
       RETURN_VALUE="${?}"
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
-      # [ -d "${CONFIG}" ] && chmod 777 "${CONFIG}" && rm -r "${CONFIG}"
+      [ -d "${CONFIG}" ] && chmod 777 "${CONFIG}" && rm -r "${CONFIG}"
       return "${RETURN_VALUE}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -201,7 +199,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -226,7 +223,8 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
+    INCORRECT_CONFIG=$(get_config_name "incorrect-")
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config_mismatch_default() {
       local TEST_NAME="${1}"
@@ -235,13 +233,13 @@ Describe 'login-negative'
       local REGISTRY="${4}"
       local USERNAME="${5}"
       local PASSWORD="${6}"
+      local INCORRECT_CONFIG="${7}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local INCORRECT_CONFIG="incorrect-${CONFIG}"
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       # Also use CONFIGS_FILE to test a explicitly-set config.
       local CONFIGS_FILE=
-      CONFIGS_FILE=$(mktemp)
+      CONFIGS_FILE=$(make_test_temp_file)
       echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}"
       # The config name on the service is different from the config name used in GANTRY_REGISTRY_CONFIG
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME}"
@@ -258,7 +256,7 @@ Describe 'login-negative'
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
       rm "${CONFIGS_FILE}"
-      docker logout "${REGISTRY}" > /dev/null
+      docker logout "${REGISTRY}" 1>/dev/null
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
       [ -d "${INCORRECT_CONFIG}" ] && rm -r "${INCORRECT_CONFIG}"
       return "${RETURN_VALUE}"
@@ -266,7 +264,7 @@ Describe 'login-negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_config_mismatch_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" "${INCORRECT_CONFIG}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -275,12 +273,12 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "${INCORRECT_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       The stderr should satisfy spec_expect_message    "${USER_LOGGED_INTO_DEFAULT}.*"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${INCORRECT_CONFIG}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -289,7 +287,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       # No --with-registry-auth, due to the incorrect configuration, image inspection failed, we did not reach the updating step.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -314,7 +311,8 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
+    INCORRECT_CONFIG=$(get_config_name "incorrect-")
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config_mismatch_no_default() {
       local TEST_NAME="${1}"
@@ -323,10 +321,10 @@ Describe 'login-negative'
       local REGISTRY="${4}"
       local USERNAME="${5}"
       local PASSWORD="${6}"
-      local INCORRECT_CONFIG="incorrect-${CONFIG}"
+      local INCORRECT_CONFIG="${7}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       # The config name on the service is different from the config name used in GANTRY_REGISTRY_CONFIG
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${INCORRECT_CONFIG}" "${SERVICE_NAME}"
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
@@ -347,7 +345,7 @@ Describe 'login-negative'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}"
+      When run test_login_config_mismatch_no_default "${TEST_NAME}" "${SERVICE_NAME}" "${AUTH_CONFIG}" "${TEST_REGISTRY}" "${TEST_USERNAME}" "${TEST_PASSWORD}" "${INCORRECT_CONFIG}"
       The status should be failure
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -356,13 +354,13 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${DEFAULT_CONFIGURATION}"
       The stderr should satisfy spec_expect_message    "${LOGGED_INTO_REGISTRY}.*${TEST_REGISTRY}.*${AUTH_CONFIG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_LOGIN_TO_REGISTRY}"
-      The stderr should satisfy spec_expect_message    "incorrect-${AUTH_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
+      The stderr should satisfy spec_expect_message    "${INCORRECT_CONFIG}.*${CONFIG_IS_NOT_A_DIRECTORY}"
       # Check warnings
       The stderr should satisfy spec_expect_message    "${THERE_ARE_NUM_CONFIGURATIONS}.*"
       # This message does not present, because we don't login with the default configuration.
       The stderr should satisfy spec_expect_no_message "${USER_LOGGED_INTO_DEFAULT}.*"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config incorrect-${AUTH_CONFIG}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${INCORRECT_CONFIG}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${SKIP_UPDATING}.*${SERVICE_NAME}.*${SKIP_REASON_MANIFEST_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${PERFORM_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -371,7 +369,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       # No --with-registry-auth, due to the incorrect configuration, image inspection failed, we did not reach the updating step.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -401,7 +398,7 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_start() {
       local TEST_NAME="${1}"
@@ -429,8 +426,8 @@ Describe 'login-negative'
       local SERVICE_NAME0="${SERVICE_NAME}-0"
       local SERVICE_NAME1="${SERVICE_NAME}-1"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       # Set GANTRY_AUTH_CONFIG_LABEL on SERVICE_NAME1, but not on SERVICE_NAME0.
       # Inspection of SERVICE_NAME0 should fail, because GANTRY_AUTH_CONFIG_LABEL is not found.
       # Inspection of SERVICE_NAME1 should pass, because configuration is set via GANTRY_AUTH_CONFIG_LABEL.
@@ -486,8 +483,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME0}"
       # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME1}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME0}.*"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME1}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME0}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME1}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME0}"
@@ -514,7 +509,7 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_REGISTRY_CONFIGS_FILE_bad_format() {
       local TEST_NAME="${1}"
@@ -525,7 +520,7 @@ Describe 'login-negative'
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       local CONFIGS_FILE=
-      CONFIGS_FILE=$(mktemp)
+      CONFIGS_FILE=$(make_test_temp_file)
       # Add an extra item to the line.
       echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD} Extra" >> "${CONFIGS_FILE}"
       # Missing an item from the line.
@@ -562,7 +557,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -587,7 +581,7 @@ Describe 'login-negative'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_file_not_exist() {
       local TEST_NAME="${1}"
@@ -631,7 +625,6 @@ Describe 'login-negative'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATING}"
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index 7aaf316..f46fc32 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -15,8 +15,8 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
-# This warning is generated by a docker command.
-export IMAGE_DIGEST_WARNING="image .* could not be accessed on a registry to record its digest. Each node will access .* independently, possibly leading to different nodes running different versions of the image"
+# FROM_DOCKER_IMAGE_DIGEST_WARNING is generated by a docker command.
+export FROM_DOCKER_IMAGE_DIGEST_WARNING="image .* could not be accessed on a registry to record its digest. Each node will access .* independently, possibly leading to different nodes running different versions of the image"
 
 Describe 'login'
   SUITE_NAME="login"
@@ -29,7 +29,7 @@ Describe 'login'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_config() {
       local TEST_NAME="${1}"
@@ -39,8 +39,8 @@ Describe 'login'
       local USERNAME="${5}"
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_REGISTRY_CONFIG="${CONFIG}"
@@ -81,7 +81,7 @@ Describe 'login'
       # 2 "--with-registry-auth" for SERVICE_NAME. One is automatically added. The other is from user.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*automatically.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*specified by user.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -113,8 +113,8 @@ Describe 'login'
       local USERNAME="${5}"
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
-      local USER_FILE=; USER_FILE=$(mktemp); echo "${USERNAME}" > "${USER_FILE}";
-      local PASS_FILE=; PASS_FILE=$(mktemp); echo "${PASSWORD}" > "${PASS_FILE}";
+      local USER_FILE=; USER_FILE=$(make_test_temp_file); echo "${USERNAME}" > "${USER_FILE}";
+      local PASS_FILE=; PASS_FILE=$(make_test_temp_file); echo "${PASSWORD}" > "${PASS_FILE}";
       # Do not set GANTRY_AUTH_CONFIG_LABEL on the service.
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       # Do not set GANTRY_REGISTRY_CONFIG to login to the default configuration.
@@ -124,7 +124,7 @@ Describe 'login'
       local RETURN_VALUE=
       run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
-      docker logout "${REGISTRY}" > /dev/null
+      docker logout "${REGISTRY}" 1>/dev/null
       rm "${USER_FILE}"
       rm "${PASS_FILE}"
       [ -d "${CONFIG}" ] && rm -r "${CONFIG}" && echo "${CONFIG} should not exist." >&2 && return 1
@@ -151,7 +151,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}.*--config.*"
       # Gantry adds --with-registry-auth for using the default configuration.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -176,7 +176,7 @@ Describe 'login'
     # When running with an Gantry image, docker buildx writes files to this folder which are owned by root.
     # Using a relative path, this the container will not write to the folder on the host.
     # So do not use an absolute path, otherwise we cannot remove this folder on the host.
-    AUTH_CONFIG="C$(unique_id)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_REGISTRY_CONFIGS_FILE() {
       local TEST_NAME="${1}"
@@ -187,7 +187,7 @@ Describe 'login'
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       local CONFIGS_FILE=
-      CONFIGS_FILE=$(mktemp)
+      CONFIGS_FILE=$(make_test_temp_file)
       echo "# Test comments: CONFIG REGISTRY USERNAME PASSWORD" >> "${CONFIGS_FILE}"
       echo "${CONFIG} ${REGISTRY} ${USERNAME} ${PASSWORD}" >> "${CONFIGS_FILE}"
       docker_service_update --label-add "${GANTRY_AUTH_CONFIG_LABEL}=${CONFIG}" "${SERVICE_NAME}"
@@ -229,7 +229,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--config ${AUTH_CONFIG}.*${SERVICE_NAME}"
       # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -251,7 +251,7 @@ Describe 'login'
     TEST_NAME="test_login_external_config"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    AUTH_CONFIG="$(mktemp -d)"
+    AUTH_CONFIG=$(get_config_name)
     TEST_REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
     test_login_external_config() {
       local TEST_NAME="${1}"
@@ -262,7 +262,7 @@ Describe 'login'
       local PASSWORD="${6}"
       check_login_input "${REGISTRY}" "${USERNAME}" "${PASSWORD}" || return 1;
       # Login outside Gantry.
-      echo "${PASSWORD}" | docker --config "${CONFIG}" login --username="${USERNAME}" --password-stdin "${REGISTRY}" > /dev/null 2>&1
+      echo "${PASSWORD}" | docker --config "${CONFIG}" login --username="${USERNAME}" --password-stdin "${REGISTRY}" 1>/dev/null 2>/dev/null
       chmod 555 "${CONFIG}"
       # Do not set GANTRY_AUTH_CONFIG_LABEL on service.
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
@@ -272,7 +272,7 @@ Describe 'login'
       # We cannot remove DOCKER_CONFIG when it containes data from root.
       export GANTRY_MANIFEST_CMD="manifest"
       export GANTRY_MANIFEST_OPTIONS="--insecure"
-      # Do not set --with-registry-auth to trigger a warning IMAGE_DIGEST_WARNING.
+      # Do not set --with-registry-auth to trigger a warning FROM_DOCKER_IMAGE_DIGEST_WARNING.
       local RETURN_VALUE=
       run_gantry "${SUITE_NAME}" "${TEST_NAME}"
       RETURN_VALUE="${?}"
@@ -301,7 +301,7 @@ Describe 'login'
       # No --with-registry-auth.
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*"
       # Check the warning due to missing --with-registry-auth.
-      The stderr should satisfy spec_expect_message    "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*${IMAGE_DIGEST_WARNING}"
+      The stderr should satisfy spec_expect_message    "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_notify_spec.sh b/tests/gantry_notify_spec.sh
index d535cb0..ad24466 100644
--- a/tests/gantry_notify_spec.sh
+++ b/tests/gantry_notify_spec.sh
@@ -34,8 +34,8 @@ _notify_before_all() {
   initialize_all_tests "${SUITE_NAME}"
   pull_image_if_not_exist caronc/apprise
   pull_image_if_not_exist axllent/mailpit
-  docker_remove "${SERVICE_NAME_APPRISE}" 1>/dev/null 2>&1
-  docker_remove "${SERVICE_NAME_MAILPIT}" 1>/dev/null 2>&1
+  docker_remove "${SERVICE_NAME_APPRISE}" 1>/dev/null 2>/dev/null
+  docker_remove "${SERVICE_NAME_MAILPIT}" 1>/dev/null 2>/dev/null
   # Use docker_run to improve coverage on lib-common.sh. `docker run` can do the same thing.
   docker_run -d --restart=on-failure:10 --name="${SERVICE_NAME_APPRISE}" --network=host \
     -e "APPRISE_STATELESS_URLS=mailto://localhost:${SMTP_PORT}?user=userid&pass=password" \
@@ -329,7 +329,8 @@ Describe 'notify'
       local SERVICE_NAME="${2}"
       local RETURN_VALUE=0
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
-      export GANTRY_UPDATE_OPTIONS="--bad-options-that-causes-error"
+      # Bad options will cause an update failure.
+      export GANTRY_UPDATE_OPTIONS="--incorrect-option"
       export GANTRY_NOTIFICATION_APPRISE_URL="http://localhost:${APPRISE_PORT}/notify"
       export GANTRY_NOTIFICATION_CONDITION="on-change"
       export GANTRY_NOTIFICATION_TITLE="TEST_TITLE"
@@ -357,7 +358,8 @@ Describe 'notify'
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
-      The stderr should satisfy spec_expect_message    "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      # Rollback should fail due to FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC
+      The stderr should satisfy spec_expect_message    "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}.*${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
diff --git a/tests/gantry_rollback_spec.sh b/tests/gantry_rollback_spec.sh
index ca2f9ea..e61a934 100644
--- a/tests/gantry_rollback_spec.sh
+++ b/tests/gantry_rollback_spec.sh
@@ -55,6 +55,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
       The stderr should satisfy spec_expect_message    "${ROLLED_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
@@ -107,6 +108,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--with-registry-auth.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
@@ -157,6 +159,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
@@ -183,7 +186,7 @@ Describe 'rollback'
       # Assume service update won't be done within TIMEOUT second.
       export GANTRY_UPDATE_TIMEOUT_SECONDS="${TIMEOUT}"
       # label should override the global environment variable.
-      export GANTRY_ROLLBACK_OPTIONS="--insecure"
+      export GANTRY_ROLLBACK_OPTIONS="--incorrect-option"
       # Rollback would fail due to the incorrect option.
       # --with-registry-auth cannot be combined with --rollback.
       local LABEL_AND_VALUE="gantry.rollback.options=--with-registry-auth"
@@ -212,6 +215,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--with-registry-auth.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
@@ -264,6 +268,7 @@ Describe 'rollback'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
       The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index 5e07452..9c0fd11 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -47,7 +47,6 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -94,7 +93,6 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -151,7 +149,6 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -203,7 +200,6 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
-      The stderr should satisfy spec_expect_no_message "${THERE_ARE_ADDITIONAL_MESSAGES}.*${SERVICE_NAME}.*"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_update_options_spec.sh b/tests/gantry_update_options_spec.sh
index 49b9495..f0c9e22 100644
--- a/tests/gantry_update_options_spec.sh
+++ b/tests/gantry_update_options_spec.sh
@@ -25,11 +25,11 @@ Describe 'update-options'
   SUITE_NAME="update-options"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_update_UPDATE_OPTIONS"
-    TEST_NAME="test_update_UPDATE_OPTIONS"
+  Describe "test_update_UPDATE_OPTIONS_good"
+    TEST_NAME="test_update_UPDATE_OPTIONS_good"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    test_update_UPDATE_OPTIONS() {
+    test_update_UPDATE_OPTIONS_good() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       local LABEL="gantry.test"
@@ -48,7 +48,7 @@ Describe 'update-options'
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_UPDATE_OPTIONS "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_update_UPDATE_OPTIONS_good "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       # Check an observable difference before and after applying UPDATE_OPTIONS.
@@ -140,6 +140,54 @@ Describe 'update-options'
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
+  Describe "test_update_UPDATE_OPTIONS_bad"
+    TEST_NAME="test_update_UPDATE_OPTIONS_bad"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
+    test_update_UPDATE_OPTIONS_bad() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      # Bad options will cause an update failure.
+      export GANTRY_UPDATE_OPTIONS="--incorrect-option"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
+    }
+    BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_update_UPDATE_OPTIONS_bad "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be failure
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
+      The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--incorrect-option.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${ROLLING_BACK}.*${SERVICE_NAME}"
+      # Rollback should fail due to FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC
+      The stderr should satisfy spec_expect_message    "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}.*${FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
+    End
+  End
   Describe "test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     TEST_NAME="test_update_UPDATE_TIMEOUT_SECONDS_not_a_number"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index c9c927e..131a8a3 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -45,7 +45,6 @@ export ADDING_OPTIONS="Adding options"
 export ADDING_OPTIONS_WITH_REGISTRY_AUTH="Adding options.*--with-registry-auth"
 export SET_TIMEOUT_TO="Set timeout to"
 export RETURN_VALUE_INDICATES_TIMEOUT="The return value [0-9]+ indicates the job timed out."
-export THERE_ARE_ADDITIONAL_MESSAGES="There are additional messages from updating"
 export NUM_SERVICES_SKIP_JOBS="Skip updating [0-9]+ service\(s\) due to they are job\(s\)"
 export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)"
 export NUM_SERVICES_NO_NEW_IMAGES="No new images for [0-9]+ service\(s\)"
@@ -54,6 +53,8 @@ export NO_UPDATES="No updates"
 export UPDATED="Updated"
 export ROLLING_BACK="Rolling back"
 export FAILED_TO_ROLLBACK="Failed to roll back"
+# FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC is generated by a docker command.
+export FROM_DOCKER_DOES_NOT_HAVE_A_PREVIOUS_SPEC="does not have a previous spec"
 export ROLLED_BACK="Rolled back"
 export NO_SERVICES_UPDATED="No services updated"
 export NUM_SERVICES_UPDATED="[0-9]+ service\(s\) updated"
@@ -69,6 +70,8 @@ export DONE_REMOVING_IMAGES="Done removing images"
 export SCHEDULE_NEXT_UPDATE_AT="Schedule next update at"
 export SLEEP_SECONDS_BEFORE_NEXT_UPDATE="Sleep [0-9]+ seconds before next update"
 
+export GANTRY_TEST_TEMP_DIR="gantry-test-tmp"
+
 test_log() {
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^NONE$"  && return 0;
   echo "${GANTRY_LOG_LEVEL}" | grep -q -i "^ERROR$"  && return 0;
@@ -235,7 +238,7 @@ _next_available_port() {
 _get_docker_config_file() {
   local REGISTRY="${1:?}"
   REGISTRY=$(echo "${REGISTRY}" | tr ':' '-')
-  echo "/tmp/gantry-test-docker-config-${REGISTRY}"
+  echo "${GANTRY_TEST_TEMP_DIR}/gantry-test-docker-config-${REGISTRY}"
 }
 
 _get_docker_config_argument() {
@@ -249,13 +252,9 @@ _get_docker_config_argument() {
 
 _login_test_registry() {
   local ENFORCE_LOGIN="${1}"
-  local REGISTRY="${2}"
-  local USERNAME="${3}"
-  local PASSWORD="${4}"
-  if ! _enforce_login_enabled "${ENFORCE_LOGIN}"; then
-    USERNAME="username"
-    PASSWORD="password"
-  fi
+  local REGISTRY="${2:?}"
+  local USERNAME="${3:?}"
+  local PASSWORD="${4:?}"
   echo "Logging in ${REGISTRY}."
   local CONFIG=
   CONFIG=$(_get_docker_config_file "${REGISTRY}") || return 1
@@ -267,17 +266,15 @@ _logout_test_registry() {
   local REGISTRY="${2}"
   local CONFIG=
   CONFIG=$(_get_docker_config_file "${REGISTRY}") || return 1
-  if _enforce_login_enabled "${ENFORCE_LOGIN}"; then
-    echo "Logging out ${REGISTRY}."
-    docker --config "${CONFIG}" logout
-  fi
+  echo "Logging out ${REGISTRY}."
+  docker --config "${CONFIG}" logout
   [ -d "${CONFIG}" ] && rm -r "${CONFIG}"
 }
 
 _get_test_registry_file() {
   local SUITE_NAME="${1:?}"
   SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-')
-  echo "/tmp/TEST_REGISTRY-${SUITE_NAME}"
+  echo "${GANTRY_TEST_TEMP_DIR}/gantry-test-registry-${SUITE_NAME}"
 }
 
 _remove_test_registry_file() {
@@ -299,6 +296,7 @@ _store_test_registry() {
   echo "${TEST_REGISTRY}" > "${REGISTRY_FILE}"
 }
 
+
 load_test_registry() {
   local SUITE_NAME="${1:?}"
   local REGISTRY_FILE=
@@ -307,6 +305,20 @@ load_test_registry() {
   cat "${REGISTRY_FILE}"
 }
 
+_get_test_registry_password_file() {
+  local SUITE_NAME="${1:?}"
+  local PASSWORD_FILE=
+  PASSWORD_FILE="$(_get_test_registry_file "${SUITE_NAME}")-password" || return 1
+  readlink -f "${PASSWORD_FILE}"
+}
+
+_remove_test_registry_password_file() {
+  local SUITE_NAME="${1:?}"
+  local PASSWORD_FILE=
+  PASSWORD_FILE=$(_get_test_registry_password_file "${SUITE_NAME}") || return 1
+  [ -r "${PASSWORD_FILE}" ] && rm "${PASSWORD_FILE}"
+}
+
 _start_registry() {
   local SUITE_NAME="${1:?}"
   local ENFORCE_LOGIN="${2}"
@@ -314,6 +326,7 @@ _start_registry() {
   SUITE_NAME=$(echo "${SUITE_NAME}" | tr ' ' '-')
   local SUITE_NAME_LENGTH="${#SUITE_NAME}"
   local REGISTRY_SERVICE_NAME="gantry-test-registry-${SUITE_NAME}"
+  REGISTRY_SERVICE_NAME=$(_sanitize_test_service_name "${REGISTRY_SERVICE_NAME}")
   local REGISTRY_BASE="localhost"
   local REGISTRY_PORT=
   REGISTRY_PORT=$(_get_initial_port "${SUITE_NAME_LENGTH}")
@@ -334,8 +347,8 @@ _start_registry() {
       echo "_start_registry _next_available_port error: REGISTRY_PORT is empty." >&2
       return 1
     fi
-    docker container stop "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
-    docker container rm -f "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
+    docker container stop "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>/dev/null;
+    docker container rm -f "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>/dev/null;
     TEST_REGISTRY="${REGISTRY_BASE}:${REGISTRY_PORT}"
     echo "Suite \"${SUITE_NAME}\" starts registry ${TEST_REGISTRY} "
     local CID=
@@ -348,7 +361,7 @@ _start_registry() {
       -e "REGISTRY_HTTP_ADDR=${TEST_REGISTRY}" \
       -e "REGISTRY_HTTP_HOST=http://${TEST_REGISTRY}" \
       --stop-timeout "${TIMEOUT_SECONDS}" \
-      $(_add_htpasswd "${ENFORCE_LOGIN}" "${TEST_USERNAME}" "${TEST_PASSWORD}") \
+      $(_add_htpasswd "${SUITE_NAME}" "${ENFORCE_LOGIN}" "${TEST_USERNAME}" "${TEST_PASSWORD}") \
       "${REGISTRY_IMAGE}" 2>&1); then
       local STATUS=
       while [ "${STATUS}" != "running" ]; do
@@ -356,6 +369,7 @@ _start_registry() {
       done
       break;
     fi
+    echo "docker container run: ${CID}";
     if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
       echo "_start_registry Reach MAX_RETRIES ${MAX_RETRIES}" >&2
       return 1
@@ -384,11 +398,13 @@ _stop_registry() {
   local REGISTRY=
   REGISTRY=$(load_test_registry "${SUITE_NAME}") || return 1
   echo "Removing registry ${REGISTRY} "
-  docker container stop "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
-  docker container rm -f "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>&1;
-  _logout_test_registry "${ENFORCE_LOGIN}" "${REGISTRY}" || return 1
-  _remove_test_registry_file "${SUITE_NAME}" || return 1
-  return 0
+  docker container stop "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>/dev/null;
+  docker container rm -f "${REGISTRY_SERVICE_NAME}" 1>/dev/null 2>/dev/null;
+  local RETURN_VALUE=0
+  _logout_test_registry "${ENFORCE_LOGIN}" "${REGISTRY}" || RETURN_VALUE=1
+  _remove_test_registry_file "${SUITE_NAME}" || RETURN_VALUE=1
+  _remove_test_registry_password_file "${SUITE_NAME}" || RETURN_VALUE=1
+  return "${RETURN_VALUE}"
 }
 
 _get_test_service_image() {
@@ -406,7 +422,7 @@ _build_and_push_gantry_image() {
   IMAGE="$(_get_gantry_image "${SUITE_NAME}")" || return 1
   pull_image_if_not_exist "$(_get_test_service_image)"
   echo "Building gantry image ${IMAGE}"
-  timeout 120 docker build --quiet -t "${IMAGE}" .
+  docker build --quiet --tag "${IMAGE}" .
   echo "Pushing gantry image ${IMAGE}"
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
@@ -417,7 +433,7 @@ _remove_gantry_image() {
   local SUITE_NAME="${1:?}"
   IMAGE="$(_get_gantry_image "${SUITE_NAME}")" || return 1
   echo "Removing gantry image ${IMAGE}"
-  docker image rm "${IMAGE}"
+  docker image rm -f "${IMAGE}"
 }
 
 initialize_all_tests() {
@@ -430,6 +446,9 @@ initialize_all_tests() {
   echo "=============================="
   echo "== Starting suite ${SUITE_NAME}"
   echo "=============================="
+  mkdir -p "${GANTRY_TEST_TEMP_DIR}"
+  uname --all
+  docker_version
   _init_swarm
   _start_registry "${SUITE_NAME}" "${ENFORCE_LOGIN}"
   _build_and_push_gantry_image "${SUITE_NAME}"
@@ -594,6 +613,21 @@ unique_id() {
   echo "${PID}-${TIME_STR}-${RANDOM_STR}"
 }
 
+make_test_temp_file() {
+  local TEMP=
+  TEMP=$(mktemp -p "${GANTRY_TEST_TEMP_DIR}")
+  TEMP=$(readlink -f "${TEMP}")
+  echo "${TEMP}"
+}
+
+get_config_name() {
+  local BASE="${1}"
+  local TEMP=
+  TEMP="${GANTRY_TEST_TEMP_DIR}/${BASE}C$(unique_id)"
+  TEMP=$(readlink -f "${TEMP}")
+  echo "${TEMP}"
+}
+
 build_test_image() {
   local IMAGE_WITH_TAG="${1}"
   local TASK_SECONDS="${2}"
@@ -609,12 +643,12 @@ build_test_image() {
     EXIT_CMD="sleep ${EXIT_SECONDS};"
   fi
   local FILE=
-  FILE=$(mktemp)
+  FILE=$(make_test_temp_file)
   echo "FROM $(_get_test_service_image)" > "${FILE}"
   echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
   pull_image_if_not_exist "$(_get_test_service_image)"
   echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
-  timeout 120 docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
+  docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
   rm "${FILE}"
 }
 
@@ -636,7 +670,7 @@ prune_local_test_image() {
 }
 
 docker_service_update() {
-  docker service update --quiet "${@}" >/dev/null
+  docker service update --quiet "${@}" 1>/dev/null
 }
 
 wait_zero_running_tasks() {
@@ -646,7 +680,7 @@ wait_zero_running_tasks() {
   local REPLICAS=
   local USED_SECONDS=0
   local TRIES=0
-  local MAX_RETRIES=120
+  local MAX_RETRIES=60
   echo "Wait until ${SERVICE_NAME} has zero running tasks."
   while [ "${NUM_RUNS}" -ne 0 ]; do
     if [ -n "${TIMEOUT_SECONDS}" ] && [ "${USED_SECONDS}" -ge "${TIMEOUT_SECONDS}" ]; then
@@ -697,10 +731,10 @@ _location_constraints() {
 
 pull_image_if_not_exist() {
   local IMAGE="${1}"
-  if docker image inspect "${IMAGE}" > /dev/null 2>&1; then
+  if docker image inspect "${IMAGE}" 1>/dev/null 2>/dev/null; then
     return 0
   fi
-  docker pull "${IMAGE}" > /dev/null
+  docker pull "${IMAGE}" 1>/dev/null
 }
 
 _enforce_login_enabled() {
@@ -709,29 +743,30 @@ _enforce_login_enabled() {
 }
 
 _add_htpasswd() {
-  local ENFORCE_LOGIN="${1}"
-  local USER="${2}"
-  local PASS="${3}"
+  local SUITE_NAME="${1:?}"
+  local ENFORCE_LOGIN="${2}"
+  local USER="${3}"
+  local PASS="${4}"
   if ! _enforce_login_enabled "${ENFORCE_LOGIN}"; then
     return 0
   fi
   local HTTPD_IMAGE="httpd:2"
   # https://distribution.github.io/distribution/about/deploying/#native-basic-auth
-  local PASSWD=
-  PASSWD="$(mktemp)"
+  local PASSWORD_FILE=
+  PASSWORD_FILE=$(_get_test_registry_password_file "${SUITE_NAME}") || return 1
   pull_image_if_not_exist "${HTTPD_IMAGE}"
-  docker_run --entrypoint htpasswd "${HTTPD_IMAGE}" -Bbn "${USER}" "${PASS}" > "${PASSWD}"
-  echo "--mount type=bind,source=${PASSWD},target=${PASSWD} \
+  docker_run --entrypoint htpasswd "${HTTPD_IMAGE}" -Bbn "${USER}" "${PASS}" > "${PASSWORD_FILE}"
+  echo "--mount type=bind,source=${PASSWORD_FILE},target=${PASSWORD_FILE} \
         -e REGISTRY_AUTH=htpasswd \
         -e REGISTRY_AUTH_HTPASSWD_REALM=RegistryRealm \
-        -e REGISTRY_AUTH_HTPASSWD_PATH=${PASSWD} "
+        -e REGISTRY_AUTH_HTPASSWD_PATH=${PASSWORD_FILE} "
 }
 
 _wait_service_state() {
   local SERVICE_NAME="${1}";
   local WANT_STATE="${2}";
   local TIMEOUT_SECONDS="${3}";
-  wait_service_state "${SERVICE_NAME}" "${WANT_STATE}" "${TIMEOUT_SECONDS}" 1>/dev/null 2>&1
+  wait_service_state "${SERVICE_NAME}" "${WANT_STATE}" "${TIMEOUT_SECONDS}" 1>/dev/null 2>/dev/null
 }
 
 _sanitize_test_service_name() {
@@ -767,7 +802,7 @@ start_replicated_service() {
   # * Add --rollback-monitor=1s: needs to exam the effect.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
-  timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
+  timeout 60 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
     --name "${SERVICE_NAME}" \
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
@@ -779,7 +814,7 @@ start_replicated_service() {
     --mode=replicated \
     --detach=true \
     "${IMAGE_WITH_TAG}";
-  _wait_service_state "${SERVICE_NAME}" "Running" 120
+  _wait_service_state "${SERVICE_NAME}" "Running" 60
 }
 
 start_multiple_replicated_services() {
@@ -808,7 +843,7 @@ start_global_service() {
   # Do not add --detach, because we want to wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
-  timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
+  timeout 60 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
     --name "${SERVICE_NAME}" \
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
@@ -831,7 +866,7 @@ _start_replicated_job() {
   # Always add --detach=true, do not wait for the job finishes.
   # SC2046 (warning): Quote this to prevent word splitting.
   # shellcheck disable=SC2046
-  timeout 120 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
+  timeout 60 docker $(_get_docker_config_argument "${IMAGE_WITH_TAG}") service create --quiet \
     --name "${SERVICE_NAME}" \
     --restart-condition "on-failure" \
     --restart-max-attempts 5 \
@@ -842,7 +877,7 @@ _start_replicated_job() {
     --detach=true \
     "${IMAGE_WITH_TAG}";
   # wait until the job is running
-  _wait_service_state "${SERVICE_NAME}" "Running" 120
+  _wait_service_state "${SERVICE_NAME}" "Running" 60
 }
 
 stop_service() {
@@ -917,7 +952,7 @@ _add_file_to_mount_options() {
     TARGET=$(readlink -f "${HOST_PATH}")
     local READONLY=
     READONLY=$(_get_file_readonly "${HOST_PATH}")
-    MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${HOST_PATH},target=${TARGET},readonly=${READONLY}"
+    MOUNT_OPTIONS="${MOUNT_OPTIONS} --mount type=bind,source=${TARGET},target=${TARGET},readonly=${READONLY}"
   fi
   echo "${MOUNT_OPTIONS}"
 }
@@ -933,7 +968,7 @@ stop_gantry_container() {
   local STACK="${1}"
   local SERVICE_NAME=
   SERVICE_NAME=$(_get_gantry_sut_name "${STACK}")
-  if ! docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" >/dev/null 2>&1; then
+  if ! docker service inspect --format '{{.ID}}' "${SERVICE_NAME}" 1>/dev/null 2>/dev/null; then
     return 0
   fi
   local RETURN_VALUE=0
@@ -947,12 +982,13 @@ stop_gantry_container() {
 }
 
 _run_gantry_container() {
-  local STACK="${1}"
-  local SUT_REPO_TAG="${2}"
+  local SUITE_NAME="${1}"
+  local STACK="${2}"
+  local SUT_REPO_TAG="${3}"
   pull_image_if_not_exist "${SUT_REPO_TAG}"
   local SERVICE_NAME=
   SERVICE_NAME=$(_get_gantry_sut_name "${STACK}")
-  docker service rm "${SERVICE_NAME}" >/dev/null 2>&1;
+  docker service rm "${SERVICE_NAME}" 1>/dev/null 2>/dev/null;
   local MOUNT_OPTIONS=
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_TEST_HOST_TO_CONTAINER}")
   MOUNT_OPTIONS=$(_add_file_to_mount_options "${MOUNT_OPTIONS}" "${GANTRY_TEST_DOCKER_CONFIG}")
@@ -1051,7 +1087,7 @@ run_gantry() {
   local SUT_REPO_TAG=
   SUT_REPO_TAG="$(_get_sut_image "${SUITE_NAME}")"
   if [ -n "${SUT_REPO_TAG}" ]; then
-    _run_gantry_container "${STACK}" "${SUT_REPO_TAG}"
+    _run_gantry_container "${SUITE_NAME}" "${STACK}" "${SUT_REPO_TAG}"
     RETURN_VALUE=$?
   else
     [ -n "${GANTRY_TEST_DOCKER_CONFIG}" ] && export DOCKER_CONFIG="${GANTRY_TEST_DOCKER_CONFIG}"

From d1a0d1ff58d8688900b79057d607c5582a878acc Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Tue, 3 Dec 2024 13:55:21 -0800
Subject: [PATCH 27/27] [docs][tests] clarify documents about authentication
 and filter. Test updating to local images.

---
 README.md                           |  4 +-
 docs/authentication.md              | 26 +++++-----
 docs/development.md                 |  4 +-
 docs/faq.md                         | 24 +++++-----
 docs/migration.md                   | 10 ++--
 src/lib-gantry.sh                   | 26 +++++++---
 tests/gantry_login_spec.sh          |  5 ++
 tests/gantry_manifest_spec.sh       | 15 +++---
 tests/gantry_service_single_spec.sh | 74 ++++++++++++++++++++++++++++-
 tests/spec_gantry_test_helper.sh    | 34 +++++++++----
 10 files changed, 166 insertions(+), 56 deletions(-)

diff --git a/README.md b/README.md
index 5e918a4..4a9bb23 100644
--- a/README.md
+++ b/README.md
@@ -69,8 +69,8 @@ You can configure the most behaviors of *Gantry* via environment variables.
 | Environment Variable  | Default | Description |
 |-----------------------|---------|-------------|
 | GANTRY_SERVICES_EXCLUDED         | | A space separated list of services names that are excluded from updating. |
-| GANTRY_SERVICES_EXCLUDED_FILTERS | `label=gantry.services.excluded=true` | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter), e.g. `label=project=project-a`. Exclude services which match the given filters from updating. The default value allows you to add label `gantry.services.excluded=true` to services to exclude them from updating. Note that multiple filters will be logical **ANDED**. |
-| GANTRY_SERVICES_FILTERS          | | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter) that are accepted by `docker service ls --filter` to select services to update, e.g. `label=project=project-a`. Note that multiple filters will be logical **ANDED**. Also see [How to filters multiple services by name](docs/faq.md#how-to-filters-multiple-services-by-name). |
+| GANTRY_SERVICES_EXCLUDED_FILTERS | `label=gantry.services.excluded=true` | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter), e.g. `label=project=project-a`. Exclude services which match the given filters from updating. The default value allows you to add label `gantry.services.excluded=true` to services to exclude them from updating. Note that multiple filters will be logical **ANDED**. An empty string means no filters, as a result *Gantry* will not exclude any services. |
+| GANTRY_SERVICES_FILTERS          | | A space separated list of [filters](https://docs.docker.com/engine/reference/commandline/service_ls/#filter) that are accepted by `docker service ls --filter` to select services to update, e.g. `label=project=project-a`. Note that multiple filters will be logical **ANDED**. An empty string means no filters, as a result *Gantry* will update all services. Also see [How to filters multiple services by name](docs/faq.md#how-to-filters-multiple-services-by-name). |
 
 > NOTE: *Gantry* reads labels on the services not on the containers. The labels need to go to the [deploy](https://docs.docker.com/reference/compose-file/deploy/#labels) section, if you are using docker compose files to setup your services.
 
diff --git a/docs/authentication.md b/docs/authentication.md
index 08e0073..228d20a 100644
--- a/docs/authentication.md
+++ b/docs/authentication.md
@@ -1,6 +1,6 @@
-## Authentication
+# Authentication
 
-### Single registry
+## Single registry
 
 If you only need to login to a single registry, you can use the environment variables `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD`, `GANTRY_REGISTRY_HOST` and `GANTRY_REGISTRY_CONFIG` to provide the authentication information. You may also use the `*_FILE` variants to pass the information through files. The files can be added to the service via [Docker secret](https://docs.docker.com/engine/swarm/secrets/).
 
@@ -10,7 +10,7 @@ If you only need to login to a single registry, you can use the environment vari
 
 > NOTE: *Gantry* uses `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER` to obtain Docker Hub rate when `GANTRY_REGISTRY_HOST` is empty or `docker.io`. You can also use their `_FILE` variants. If either password or user is empty, *Gantry* reads the Docker Hub rate for anonymous users.
 
-### Multiple registries
+## Multiple registries
 
 If the images of services are hosted on multiple registries that are required authentication, you should provide a configuration file to the *Gantry* and set `GANTRY_REGISTRY_CONFIGS_FILE` correspondingly. You can use [Docker secret](https://docs.docker.com/engine/swarm/secrets/) to provision the configuration file. The configuration file must be in the following format:
 
@@ -30,17 +30,19 @@ If the images of services are hosted on multiple registries that are required au
 
 You can use `GANTRY_REGISTRY_CONFIGS_FILE` together with other authentication environment variables.
 
-You can login to multiple registries using the same Docker configuration. However if you login to the same registry with different user names for different services, you should use different Docker configurations.
+You can login to multiple registries using the same Docker configuration. For example you can set all the configurations to the default Docker configuration location `${HOME}/.docker/`. However if you login to the same registry with different user names for different services, you need to use different Docker configurations.
 
-### Select Docker configurations for services
+## Selecting Docker configurations for services
 
-If you login to a single registry using `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_HOST` without setting `GANTRY_REGISTRY_CONFIG`, the default Docker configuration is used. You don't need to set anything extra for authentication.
+If you login to a single registry using `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_HOST` **without** setting `GANTRY_REGISTRY_CONFIG`, the default Docker configuration is used. When using the default Docker configuration, you don't need to set anything extra for authentication.
 
- *Gantry* creates or updates the docker configurations based on the locations set via `GANTRY_REGISTRY_CONFIG`, `GANTRY_REGISTRY_CONFIG_FILE` or `GANTRY_REGISTRY_CONFIGS_FILE`.
+ *Gantry* creates or updates the docker configurations based on the locations set via `GANTRY_REGISTRY_CONFIG`, `GANTRY_REGISTRY_CONFIG_FILE` or `GANTRY_REGISTRY_CONFIGS_FILE`. They could be same as the the default Docker configuration location. When using the default Docker configuration, you don't need to set anything extra for authentication.
 
- You can use environment variable [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables) to apply the same Docker configuration to all docker commands, i.e. to all services.
+The default Docker configuration location is `/root/.docker/` inside the container created based on the image built from this repository, because the default user is `root`. You can use environment variable [`DOCKER_CONFIG`](https://docs.docker.com/engine/reference/commandline/cli/#environment-variables) to explicitly set the default docker configuration location, which applies to all docker commands, i.e. to all services.
 
-You need to add label `gantry.auth.config=<configuration>` to particular services to tell which Docker configuration to use for authentication, when you use different configurations for different registries. When *Gantry* finds the label `gantry.auth.config=<configuration>` on services, it adds `--config <configuration>` to the Docker commands for the corresponding services.
+Optionally you can use different configurations for different services, for example when you want to login to the same registry with different user names. In this case, besides using different configuration values in `GANTRY_REGISTRY_CONFIG` and `GANTRY_REGISTRY_CONFIG_FILE`, you need to add the label `gantry.auth.config=<configuration>` on the particular services to tell which Docker configuration to use for authentication. When *Gantry* finds the label `gantry.auth.config=<configuration>` on services, it adds `--config <configuration>` to the Docker commands for the corresponding services to overrides the default configuration location.
+
+## Adding `--with-registry-auth`
 
 *Gantry* automatically adds `--with-registry-auth` to the `docker service update` command for services for the following cases.
 
@@ -49,9 +51,9 @@ You need to add label `gantry.auth.config=<configuration>` to particular service
   * when `GANTRY_REGISTRY_USER`, `GANTRY_REGISTRY_PASSWORD` are set, while `GANTRY_REGISTRY_CONFIG` is empty.
   * when the configuration from `GANTRY_REGISTRY_CONFIG` or `GANTRY_REGISTRY_CONFIGS_FILE` is same as the default Docker configuration location `${HOME}/.docker/` or the location specified by `DOCKER_CONFIG`.
 
-You can manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` if it is not added automatically for your case. Without `--with-registry-auth`, the service will be [updated to an image without digest](https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336), and you might get a warning "*image \<image\> could not be accessed on a registry to record its digest. Each node will access \<image\> independently, possibly leading to different nodes running different versions of the image.*"
+You can manually add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` if it is not added automatically for your case. When `--with-registry-auth` is missing but the registry requires authentication, the service will be [updated to an image without digest](https://github.com/shizunge/gantry/issues/53#issuecomment-2348376336), and you will get a warning *"image \<image\> could not be accessed on a registry to record its digest. Each node will access \<image\> independently, possibly leading to different nodes running different versions of the image."*
 
-### Use an existing Docker configuration
+## Using an existing Docker configuration
 
 You can use an existing Docker configuration from the host machines for authorization when you run *Gantry* as a Docker service. You need to do the followings.
 
@@ -60,4 +62,4 @@ You can use an existing Docker configuration from the host machines for authoriz
 * Set the environment variable `DOCKER_CONFIG` on the *Gantry* container to specify the location of the Docker configuration folder inside the container. You can skip this step when you mount the folder to the default Docker configuration location `/root/.docker/` inside the container.
 * Add `--with-registry-auth` to `GANTRY_UPDATE_OPTIONS` manually.
 
-> Note that [`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/) writes data to the Docker configuration folder `${DOCKER_CONFIG}/buildx`, which therefore needs to be writable. You can set `GANTRY_MANIFEST_CMD` to `manifest` to avoid writing to the Docker configuration folder.
+> Note that [`docker buildx imagetools inspect`](https://docs.docker.com/engine/reference/commandline/buildx_imagetools_inspect/) writes data to the Docker configuration folder `${DOCKER_CONFIG}/buildx`, which therefore needs to be writable. If you want to use `buildx` and mount the configuration files read-only, you could just mount the file `config.json` and leave the folder writeable. If you have to mount the entire folder read-only, you can set `GANTRY_MANIFEST_CMD` to `manifest` to avoid writing to the Docker configuration folder. Also see [Which `GANTRY_MANIFEST_CMD` to use](../docs/faq.md#which-gantry_manifest_cmd-to-use).
diff --git a/docs/development.md b/docs/development.md
index 1ddab48..3a00aec 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -1,6 +1,6 @@
-## Development
+# Development
 
-### Internal Configurations
+## Internal Configurations
 
 The following configurations are set automatically by *Gantry* internally. User usually does not need to touch them.
 
diff --git a/docs/faq.md b/docs/faq.md
index 303daad..0c9e468 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -1,6 +1,6 @@
-## FAQ
+# FAQ
 
-### How does *Gantry* work?
+## How does *Gantry* work?
 
 Fundamentally *Gantry* calls [`docker service update`](https://docs.docker.com/engine/reference/commandline/service_update/) CLI and let docker engine [applies rolling updates to a service](https://docs.docker.com/engine/swarm/swarm-tutorial/rolling-update/).
 
@@ -8,27 +8,29 @@ Before updating a service, *Gantry* will try to obtain the manifest of the image
 
 At the end of updating, *Gantry* optionally removes the old images.
 
-### How to update standalone docker containers?
+## How to update standalone docker containers?
 
 *Gantry* only works for docker swarm services. If you need to update standalone docker containers, you can try [*watchtower*](https://github.com/containrrr/watchtower). *Gantry* can launch *watchtower* via `GANTRY_PRE_RUN_CMD` or `GANTRY_POST_RUN_CMD`. See the [example](../examples/prune-and-watchtower).
 
-### How to filters multiple services by name?
+## How to filters multiple services by name?
 
 It will not work by setting multiple filters with different names, because filters are logical **ANDED**.
 
 To filter multiple services, you can set a label on each service then let *Gantry* filter on that label via `GANTRY_SERVICES_FILTERS`. Or you can run multiple *Gantry* instances.
 
-### How to run *Gantry* on a cron schedule?
+Setting `GANTRY_SERVICES_FILTERS` to an empty string means no filters, as a result *Gantry* will update all services.
+
+## How to run *Gantry* on a cron schedule?
 
 You can start *Gantry* as a docker swarm service and use [`swarm-cronjob`](https://github.com/crazy-max/swarm-cronjob) to run it at a given time. When use `swarm-cronjob`, you need to set `GANTRY_SLEEP_SECONDS` to 0. See the [example](../examples/cronjob).
 
-### How to update services with no running tasks?
+## How to update services with no running tasks?
 
 As discussed [here](https://github.com/docker/cli/issues/627), the CLI will hang when running `docker service update` on a service with no running tasks. We must add `--detach=true` option to the `docker service update`.
 
 *Gantry* will check whether there are running tasks in a service. If there is no running task, *Gantry* automatically adds the option `--detach=true`. In addition to the detach option, *Gantry* also adds `--replicas=0` for services in replicated mode. You don't need to add these options manually.
 
-### Which `GANTRY_MANIFEST_CMD` to use?
+## Which `GANTRY_MANIFEST_CMD` to use?
 
 Before updating a service, *Gantry* will try to obtain the image's meta data to decide whether there is a new image. If there is no new image, *Gantry* skips calling `docker service update`, leading to a speedup of the overall process.
 
@@ -36,14 +38,14 @@ Before updating a service, *Gantry* will try to obtain the image's meta data to
 
 `manifest` is kept for debugging purpose. The only known advantage of [`docker manifest inspect`](https://docs.docker.com/engine/reference/commandline/manifest_inspect/) is that it does not require the write permission to the [Docker configuration folder](https://docs.docker.com/engine/reference/commandline/cli/#configuration-files). If the Docker configuration folder is read-only, you need to use `manifest` to avoid permission deny errors.
 
-`none` can be used to disable the image inspection. One use case of `none` is that you want to add `--force` to the `docker service update` command via `GANTRY_UPDATE_OPTIONS`, which updates the services even if there is nothing changed. Another use case is to debug image inspection. Please report the bug through a [GitHub issue](https://github.com/shizunge/gantry/issues), thanks.
+`none` can be used to disable the image inspection. One use case of `none` is that you want to update services using local built images that are not available on a registry, in which case you want to add `--force` to the `docker service update` command via `GANTRY_UPDATE_OPTIONS`. `none` can also be used to debug image inspection.
 
-### Can *Gantry* report Docker Hub rate for non-anonymous account?
+## Can *Gantry* report Docker Hub rate for non-anonymous account?
 
 When checking Docker Hub rate, *Gantry* reads the Docker Hub credential only from `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER`, or their `_FILE` variants. `GANTRY_REGISTRY_HOST` or its `_FILE` variant must be either empty or `docker.io`.
 
 If you need to login to multiple registries, you can use `GANTRY_REGISTRY_CONFIGS_FILE` together with `GANTRY_REGISTRY_PASSWORD` and `GANTRY_REGISTRY_USER`. Credentials in `GANTRY_REGISTRY_CONFIGS_FILE` will be used for services updating, but they won't be used for checking Docker Hub rate. See [Authentication](../docs/authentication.md) for more information.
 
-### How to use authorization from the host machines?
+## How to use authorization from the host machines?
 
-See [use an existing Docker configuration](../docs/authentication.md#use-an-existing-docker-configuration).
+See [using an existing Docker configuration](../docs/authentication.md#using-an-existing-docker-configuration).
diff --git a/docs/migration.md b/docs/migration.md
index f1af29d..3580144 100644
--- a/docs/migration.md
+++ b/docs/migration.md
@@ -1,4 +1,4 @@
-## Migration from shepherd
+# Migration from shepherd
 
 *Gantry* started to fix the following problems I found in [*shepherd*](https://github.com/containrrr/shepherd), then it became refactored and totally rewritten, with [abundant tests](../tests/README.md).
 
@@ -15,7 +15,7 @@ Although I have tried to keep backward compatibility, not all configurations in
 
 This guide helps you migrate from *shepherd* to *gantry* by highlighting the difference between them. Please refer to the [README](../README.md) for the full description of the configurations.
 
-### Equivalent or similar configurations
+## Equivalent or similar configurations
 
 | *Shepherd* Env | Equivalent or similar *Gantry* Env  | Enhancement |
 |----------------|-------------------------------------|-------------|
@@ -37,7 +37,7 @@ This guide helps you migrate from *shepherd* to *gantry* by highlighting the dif
 
 The label on the services to select config to enable authentication is renamed to `gantry.auth.config`.
 
-### Deprecated configurations
+## Deprecated configurations
 
 | *Shepherd* Env | Workaround |
 |----------------|------------|
@@ -46,7 +46,7 @@ The label on the services to select config to enable authentication is renamed t
 | WITH_NO_RESOLVE_IMAGE  | Manually add `--no-resolve-image` to `GANTRY_UPDATE_OPTIONS`. |
 | RUN_ONCE_AND_EXIT      | Set `GANTRY_SLEEP_SECONDS` to 0. |
 
-### New configurations
+## New configurations
 
 You can enable these new features of *Gantry* through new configurations.
 
@@ -74,7 +74,7 @@ You can enable these new features of *Gantry* through new configurations.
 
 Besides the global configurations via environment variables, you can apply a different value to a particular service via [labels](../README.md#labels).
 
-### License
+## License
 
 *Shepherd* is under [MIT license](https://github.com/containrrr/shepherd/blob/master/LICENSE)
 
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
index f573b8f..a9d516d 100755
--- a/src/lib-gantry.sh
+++ b/src/lib-gantry.sh
@@ -1037,18 +1037,30 @@ _update_single_service() {
     _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATE_FAILED "${SERVICE_NAME}"
     return 1
   fi
-  local TIME_ELAPSED=
-  TIME_ELAPSED=$(time_elapsed_since "${START_TIME}")
-  local PREVIOUS_IMAGE=
-  local CURRENT_IMAGE=
+  local PREVIOUS_IMAGE PREVIOUS_DIGEST
   PREVIOUS_IMAGE=$(_get_service_previous_image "${SERVICE_NAME}")
+  PREVIOUS_DIGEST=$(extract_string "${PREVIOUS_IMAGE}" '@' 2)
+  [ -z "${PREVIOUS_DIGEST}" ] && log DEBUG "After updating, the previous image ${PREVIOUS_IMAGE} of ${SERVICE_NAME} does not have a digest."
+  local CURRENT_IMAGE CURRENT_DIGEST
   CURRENT_IMAGE=$(_get_service_image "${SERVICE_NAME}")
+  CURRENT_DIGEST=$(extract_string "${CURRENT_IMAGE}" '@' 2)
+  [ -z "${CURRENT_DIGEST}" ] && log WARN "After updating, the current image ${CURRENT_IMAGE} of ${SERVICE_NAME} does not have a digest."
+  local TIME_ELAPSED=
+  TIME_ELAPSED=$(time_elapsed_since "${START_TIME}")
   if [ "${PREVIOUS_IMAGE}" = "${CURRENT_IMAGE}" ]; then
-    log INFO "No updates for ${SERVICE_NAME}. Use ${TIME_ELAPSED}."
-    return 0
+    # The same new and old images indicate that the image is still being used.
+    # Removing image would fail due to that.
+    if [ -z "${UPDATE_OPTIONS}" ]; then
+      # Unless we add more options like `--force`, docker may not really update the service due to no changes.
+      log INFO "No updates for ${SERVICE_NAME}. Use ${TIME_ELAPSED}."
+      return 0
+    fi
+    # This (e.g. no digest in both old and new image.) could happen when the service is updated to a local built image.
+  else
+    # Remove PREVIOUS_IMAGE only when it is no longer used.
+    _static_variable_add_unique_to_list STATIC_VAR_IMAGES_TO_REMOVE "${PREVIOUS_IMAGE}"
   fi
   _static_variable_add_unique_to_list STATIC_VAR_SERVICES_UPDATED "${SERVICE_NAME}"
-  _static_variable_add_unique_to_list STATIC_VAR_IMAGES_TO_REMOVE "${PREVIOUS_IMAGE}"
   log INFO "Updated ${SERVICE_NAME}. Use ${TIME_ELAPSED}."
   return 0
 }
diff --git a/tests/gantry_login_spec.sh b/tests/gantry_login_spec.sh
index f46fc32..003bc06 100644
--- a/tests/gantry_login_spec.sh
+++ b/tests/gantry_login_spec.sh
@@ -82,6 +82,7 @@ Describe 'login'
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*automatically.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*specified by user.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -152,6 +153,7 @@ Describe 'login'
       # Gantry adds --with-registry-auth for using the default configuration.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -230,6 +232,7 @@ Describe 'login'
       # Gantry adds --with-registry-auth for finding GANTRY_AUTH_CONFIG_LABEL on the service.
       The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -302,6 +305,8 @@ Describe 'login'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS_WITH_REGISTRY_AUTH}.*"
       # Check the warning due to missing --with-registry-auth.
       The stderr should satisfy spec_expect_message    "${FROM_DOCKER_IMAGE_DIGEST_WARNING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${AFTER_UPDATING_PREVIOUS_IMAGE}"
+      The stderr should satisfy spec_expect_message    "${AFTER_UPDATING_CURRENT_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_manifest_spec.sh b/tests/gantry_manifest_spec.sh
index 8fee3c2..704527d 100644
--- a/tests/gantry_manifest_spec.sh
+++ b/tests/gantry_manifest_spec.sh
@@ -19,28 +19,28 @@ Describe 'manifest-command'
   SUITE_NAME="manifest-command"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_MANIFEST_CMD_none"
-    TEST_NAME="test_MANIFEST_CMD_none"
+  Describe "test_MANIFEST_CMD_none_force"
+    TEST_NAME="test_MANIFEST_CMD_none_force"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
-    test_MANIFEST_CMD_none() {
+    test_MANIFEST_CMD_none_force() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
       export GANTRY_MANIFEST_CMD="none"
-      export GANTRY_UPDATE_OPTIONS="--force"
+      # Test that Gantry reports "no updates", when we don't add `--force` to GANTRY_UPDATE_OPTIONS while the image does not change.
       run_gantry "${SUITE_NAME}" "${TEST_NAME}"
     }
     BeforeEach "common_setup_no_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_MANIFEST_CMD_none "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_MANIFEST_CMD_none_force "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
-      # Do not set GANTRY_SERVICES_SELF, it should be set autoamtically
+      # Do not set GANTRY_SERVICES_SELF, it should be set automatically.
       # If we are not testing gantry inside a container, it should failed to find the service name.
       # To test gantry container, we need to use run_gantry_container.
       The stderr should satisfy spec_expect_no_message ".*GANTRY_SERVICES_SELF.*"
@@ -52,7 +52,8 @@ Describe 'manifest-command'
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
       The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
-      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--force.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
+      # Gantry reports no updates, unless we add `--force` to the options.
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/gantry_service_single_spec.sh b/tests/gantry_service_single_spec.sh
index 9c0fd11..24e8490 100644
--- a/tests/gantry_service_single_spec.sh
+++ b/tests/gantry_service_single_spec.sh
@@ -47,6 +47,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
+      The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_no_message "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -93,6 +94,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
+      The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -119,6 +121,7 @@ Describe 'service-single-service'
       local TEST_NAME="${1}"
       local IMAGE_WITH_TAG="${2}"
       local SERVICE_NAME="${3}"
+      initialize_test "${TEST_NAME}"
       # Start a service with image not available on the registry, the digest will not be available.
       build_test_image "${IMAGE_WITH_TAG}"
       start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 2>&1
@@ -149,6 +152,8 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
+      The stderr should satisfy spec_expect_message    "${AFTER_UPDATING_PREVIOUS_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}"
+      The stderr should satisfy spec_expect_no_message "${AFTER_UPDATING_CURRENT_IMAGE}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
@@ -161,13 +166,79 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${NO_IMAGES_TO_REMOVE}"
       The stderr should satisfy spec_expect_message    "${REMOVING_NUM_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
-      # Failed to removing the old image due to it has no digest.
+      # Failed to remove the old image due to it has no digest.
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
     End
   End
+  Describe "test_new_image_local"
+    TEST_NAME="test_new_image_local"
+    IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
+    SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
+    test_start() {
+      local TEST_NAME="${1}"
+      local IMAGE_WITH_TAG="${2}"
+      local SERVICE_NAME="${3}"
+      initialize_test "${TEST_NAME}"
+      # Start a service with image not available on the registry, the digest will not be available.
+      build_test_image "${IMAGE_WITH_TAG}"
+      start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}" 2>&1
+      # Build a new local image. Do not push to registry to test `--no-resolve-image`.
+      build_test_image "${IMAGE_WITH_TAG}"
+    }
+    test_new_image_local() {
+      local TEST_NAME="${1}"
+      local SERVICE_NAME="${2}"
+      reset_gantry_env "${SUITE_NAME}" "${SERVICE_NAME}"
+      # As the image is not pushed to the registry, we need to skip the image inspection.
+      export GANTRY_MANIFEST_CMD="none"
+      # Because both new and old images have no digest, `docker service update` thinks there is nothing to update, thus it will not run the update without `--force`.
+      export GANTRY_UPDATE_OPTIONS="--force"
+      run_gantry "${SUITE_NAME}" "${TEST_NAME}"
+    }
+    BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
+    It 'run_test'
+      When run test_new_image_local "${TEST_NAME}" "${SERVICE_NAME}"
+      The status should be success
+      The stdout should satisfy display_output
+      The stdout should satisfy spec_expect_no_message ".+"
+      The stderr should satisfy display_output
+      # Gantry is still trying to update the service.
+      # It will see a new local image.
+      The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_MANIFEST_CMD_IS_NONE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_INSPECT_FAILURE}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_NO_NEW_IMAGES}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATING}"
+      The stderr should satisfy spec_expect_message    "${ADDING_OPTIONS}.*--force.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
+      The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
+      The stderr should satisfy spec_expect_message    "${AFTER_UPDATING_PREVIOUS_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}"
+      The stderr should satisfy spec_expect_message    "${AFTER_UPDATING_CURRENT_IMAGE}.*${SERVICE_NAME}.*${DOES_NOT_HAVE_A_DIGEST}"
+      # Since we add `--force` to the option, Gantry should report that service gets updated.
+      The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_ROLLBACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${ROLLED_BACK}.*${SERVICE_NAME}"
+      The stderr should satisfy spec_expect_no_message "${NO_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_message    "${NUM_SERVICES_UPDATED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_UPDATE_FAILED}"
+      The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_ERRORS}"
+      # No removing images because we should find that new and old images are same, which indicates the images are still being used.
+      The stderr should satisfy spec_expect_message    "${NO_IMAGES_TO_REMOVE}"
+      The stderr should satisfy spec_expect_no_message "${REMOVING_NUM_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
+      The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
+    End
+  End
   Describe "test_new_image_SERVICES_SELF"
     TEST_NAME="test_new_image_SERVICES_SELF"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
@@ -200,6 +271,7 @@ Describe 'service-single-service'
       The stderr should satisfy spec_expect_no_message "${ADDING_OPTIONS}"
       The stderr should satisfy spec_expect_no_message "${SET_TIMEOUT_TO}"
       The stderr should satisfy spec_expect_no_message "${RETURN_VALUE_INDICATES_TIMEOUT}"
+      The stderr should satisfy spec_expect_no_message "${DOES_NOT_HAVE_A_DIGEST}"
       The stderr should satisfy spec_expect_message    "${UPDATED}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${NO_UPDATES}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_no_message "${ROLLING_BACK}.*${SERVICE_NAME}"
diff --git a/tests/spec_gantry_test_helper.sh b/tests/spec_gantry_test_helper.sh
index 131a8a3..cd2611e 100644
--- a/tests/spec_gantry_test_helper.sh
+++ b/tests/spec_gantry_test_helper.sh
@@ -50,6 +50,9 @@ export NUM_SERVICES_INSPECT_FAILURE="Failed to inspect [0-9]+ service\(s\)"
 export NUM_SERVICES_NO_NEW_IMAGES="No new images for [0-9]+ service\(s\)"
 export NUM_SERVICES_UPDATING="Updating [0-9]+ service\(s\)"
 export NO_UPDATES="No updates"
+export AFTER_UPDATING_CURRENT_IMAGE="After updating, the current image"
+export AFTER_UPDATING_PREVIOUS_IMAGE="After updating, the previous image"
+export DOES_NOT_HAVE_A_DIGEST="does not have a digest"
 export UPDATED="Updated"
 export ROLLING_BACK="Rolling back"
 export FAILED_TO_ROLLBACK="Failed to roll back"
@@ -642,14 +645,27 @@ build_test_image() {
   if [ -n "${EXIT_SECONDS}" ] && [ "${EXIT_SECONDS}" -gt "0" ]; then
     EXIT_CMD="sleep ${EXIT_SECONDS};"
   fi
-  local FILE=
-  FILE=$(make_test_temp_file)
-  echo "FROM $(_get_test_service_image)" > "${FILE}"
-  echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
-  pull_image_if_not_exist "$(_get_test_service_image)"
-  echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
-  docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" .
-  rm "${FILE}"
+  local RETURN_VALUE=1
+  local TRIES=0
+  local MAX_RETRIES=60
+  while [ "${RETURN_VALUE}" != "0" ]; do
+    if [ "${TRIES}" -ge "${MAX_RETRIES}" ]; then
+      echo "build_test_image Reach MAX_RETRIES ${MAX_RETRIES}" >&2
+      return 1
+    fi
+    TRIES=$((TRIES+1))
+    local FILE=
+    FILE=$(make_test_temp_file)
+    echo "FROM $(_get_test_service_image)" > "${FILE}"
+    echo "ENTRYPOINT [\"sh\", \"-c\", \"echo $(unique_id); trap \\\"${EXIT_CMD}\\\" HUP INT TERM; ${TASK_CMD}\"]" >> "${FILE}"
+    pull_image_if_not_exist "$(_get_test_service_image)"
+    echo "Building image ${IMAGE_WITH_TAG} from ${FILE}"
+    docker build --quiet --tag "${IMAGE_WITH_TAG}" --file "${FILE}" . 2>&1
+    RETURN_VALUE=$?
+    rm "${FILE}"
+    [ "${RETURN_VALUE}" != "0" ] && sleep 1
+  done
+  return 0
 }
 
 build_and_push_test_image() {
@@ -666,7 +682,7 @@ build_and_push_test_image() {
 prune_local_test_image() {
   local IMAGE_WITH_TAG="${1}"
   echo "Removing image ${IMAGE_WITH_TAG}"
-  docker image rm "${IMAGE_WITH_TAG}" --force
+  docker image rm "${IMAGE_WITH_TAG}" --force 2>&1
 }
 
 docker_service_update() {