[gantry] follow the image remover's log and eval_cmd stdout.

In tests, check DONE_REMOVING_IMAGES to ensure all logs are printed.
shizunge · Nov 23, 2024 · 794c204 · 794c204
1 parent 884787e
commit 794c204
Show file tree

Hide file tree

Showing 17 changed files with 295 additions and 161 deletions.
diff --git a/src/lib-common.sh b/src/lib-common.sh
diff --git a/src/lib-gantry.sh b/src/lib-gantry.sh
@@ -449,15 +449,14 @@ _remove_images() {
   # SC2086: Double quote to prevent globbing and word splitting.
   # shellcheck disable=SC2086
   docker_global_job --name "${SERVICE_NAME}" \
+    --detach \
     --restart-condition on-failure \
     --restart-max-attempts 1 \
     --mount type=bind,source=/var/run/docker.sock,destination=/var/run/docker.sock \
     --env "GANTRY_IMAGES_TO_REMOVE=${IMAGES_TO_REMOVE_LIST}" \
     ${CLEANUP_IMAGES_OPTIONS} \
     "${IMAGES_REMOVER}";
-  wait_service_state "${SERVICE_NAME}"
-  docker_service_logs "${SERVICE_NAME}"
-  docker_service_remove "${SERVICE_NAME}"
+  docker_service_follow_logs_wait_complete "${SERVICE_NAME}"
 }
 
 _report_list() {

diff --git a/tests/gantry_cleanup_images_spec.sh b/tests/gantry_cleanup_images_spec.sh
@@ -60,6 +60,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_CLEANUP_IMAGES_OPTIONS_bad" "container_test:true" "coverage:true"
@@ -106,6 +107,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "Failed.*--incorrect-option"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_CLEANUP_IMAGES_OPTIONS_good" "container_test:true" "coverage:true"
@@ -151,6 +153,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_no_message "Failed.*--container-label=test"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_IMAGES_TO_REMOVE_none_empty" "container_test:true" "coverage:true"
@@ -221,6 +224,7 @@ Describe 'cleanup-images'
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG0}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG1}"
       The stderr should satisfy spec_expect_message    "There is no image.*${IMAGE_WITH_TAG2}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Single service'
diff --git a/tests/gantry_common_options_spec.sh b/tests/gantry_common_options_spec.sh
@@ -64,6 +64,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
@@ -146,8 +147,10 @@ Describe 'common-options'
       export GANTRY_CLEANUP_IMAGES=
       # Test that pre-run command can change the global configurations.
       export GANTRY_PRE_RUN_CMD="echo \"Pre update\"; GANTRY_UPDATE_OPTIONS=--detach=true; GANTRY_CLEANUP_IMAGES=false;"
+      # This command outputs multiple lines.
+      local POST_CMD="for I in \$(seq 3 5); do echo \"OUTPUT_LINE=\$I\"; done"
       # Test that the command returns a non-zero value.
-      export GANTRY_POST_RUN_CMD="echo \"Post update\"; false;"
+      export GANTRY_POST_RUN_CMD="echo \"Post update\"; ${POST_CMD}; false;"
       run_gantry "${TEST_NAME}"
     }
     BeforeEach "common_setup_new_image ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
@@ -160,7 +163,7 @@ Describe 'common-options'
       The stderr should satisfy display_output
       The stderr should satisfy spec_expect_no_message "${START_WITHOUT_A_SQUARE_BRACKET}"
       The stderr should satisfy spec_expect_message    "Pre update"
-      The stderr should satisfy spec_expect_no_message "pre-run command returned a non-zero value"
+      The stderr should satisfy spec_expect_message    "Finish pre-run command."
       The stderr should satisfy spec_expect_no_message "${SKIP_UPDATING}.*${SERVICE_NAME}"
       The stderr should satisfy spec_expect_message    "${PERFORM_UPDATING}.*${SERVICE_NAME}.*${PERFORM_REASON_HAS_NEWER_IMAGE}"
       The stderr should satisfy spec_expect_no_message "${NUM_SERVICES_SKIP_JOBS}"
@@ -182,8 +185,12 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "Post update"
-      The stderr should satisfy spec_expect_message    "post-run command returned a non-zero value"
+      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=3"
+      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=4"
+      The stderr should satisfy spec_expect_message    "OUTPUT_LINE=5"
+      The stderr should satisfy spec_expect_message    "Finish post-run command with a non-zero return value 1."
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End
@@ -236,6 +243,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message        "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message        "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message        "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message        "${DONE_REMOVING_IMAGES}"
       # Check messages between iterations.
       The stderr should satisfy spec_expect_message           "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_message           "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
@@ -283,6 +291,7 @@ Describe 'common-options'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${SCHEDULE_NEXT_UPDATE_AT}"
       The stderr should satisfy spec_expect_no_message "${SLEEP_SECONDS_BEFORE_NEXT_UPDATE}"
     End

diff --git a/tests/gantry_filters_spec.sh b/tests/gantry_filters_spec.sh
@@ -61,6 +61,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_SERVICES_EXCLUDED_multiple_services" "container_test:true" "coverage:true"
@@ -118,6 +119,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_SERVICES_EXCLUDED_FILTERS_default" "container_test:true" "coverage:true"
@@ -168,6 +170,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_SERVICES_EXCLUDED_FILTERS_bad" "container_test:false" "coverage:true"
@@ -212,6 +215,7 @@ Describe 'filters'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'Filters'
diff --git a/tests/gantry_jobs_spec.sh b/tests/gantry_jobs_spec.sh
@@ -19,12 +19,12 @@ Describe 'update-jobs'
   SUITE_NAME="update-jobs"
   BeforeAll "initialize_all_tests ${SUITE_NAME}"
   AfterAll "finish_all_tests ${SUITE_NAME}"
-  Describe "test_update_jobs_skipping" "container_test:true" "coverage:true"
+  Describe "test_jobs_skipping" "container_test:true" "coverage:true"
     # For `docker service ls --filter`, the name filter matches on all or the prefix of a service's name
     # See https://docs.docker.com/engine/reference/commandline/service_ls/#name
     # It does not do the exact match of the name. See https://github.com/moby/moby/issues/32985
     # This test also checks whether we do an extra step to to perform the exact match.
-    TEST_NAME="test_update_jobs_skipping"
+    TEST_NAME="test_jobs_skipping"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     SERVICE_NAME_SUFFIX="${SERVICE_NAME}-suffix"
@@ -36,7 +36,7 @@ Describe 'update-jobs'
       common_setup_job "${TEST_NAME}" "${IMAGE_WITH_TAG}" "${SERVICE_NAME_SUFFIX}"
       start_replicated_service "${SERVICE_NAME}" "${IMAGE_WITH_TAG}"
     }
-    test_update_jobs_skipping() {
+    test_jobs_skipping() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -53,7 +53,7 @@ Describe 'update-jobs'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     AfterEach "test_end ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_skipping "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -83,16 +83,17 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_UPDATE_JOBS_true"
+  Describe "test_jobs_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+    TEST_NAME="test_jobs_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TASK_SECONDS=-1
     # Use a long EXIT_SECONDS, because we want the task keep running for a while after updating to trigger the image removing failure.
     EXIT_SECONDS=30
-    test_update_jobs_UPDATE_JOBS_true() {
+    test_jobs_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -104,7 +105,7 @@ Describe 'update-jobs'
     BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -133,16 +134,17 @@ Describe 'update-jobs'
       # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_label_UPDATE_JOBS_true"
+  Describe "test_jobs_label_UPDATE_JOBS_true" "container_test:true" "coverage:true"
+    TEST_NAME="test_jobs_label_UPDATE_JOBS_true"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     TASK_SECONDS=-1
     # Use a long EXIT_SECONDS, because we want the task keep running for a while after updating to trigger the image removing failure.
     EXIT_SECONDS=30
-    test_update_jobs_label_UPDATE_JOBS_true() {
+    test_jobs_label_UPDATE_JOBS_true() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -160,7 +162,7 @@ Describe 'update-jobs'
     BeforeEach "common_setup_job ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS} ${EXIT_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_label_UPDATE_JOBS_true "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -189,10 +191,11 @@ Describe 'update-jobs'
       # Since the job may not reach the desired state, they are still using the image. Image remover will fail.
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_message    "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
-  Describe "test_update_jobs_no_running_tasks" "container_test:true" "coverage:true"
-    TEST_NAME="test_update_jobs_no_running_tasks"
+  Describe "test_jobs_no_running_tasks" "container_test:true" "coverage:true"
+    TEST_NAME="test_jobs_no_running_tasks"
     IMAGE_WITH_TAG=$(get_image_with_tag "${SUITE_NAME}")
     SERVICE_NAME=$(get_test_service_name "${TEST_NAME}")
     # Use a short TASK_SECONDS, because we want the task finishes soon.
@@ -206,7 +209,7 @@ Describe 'update-jobs'
       # The tasks should exit after TASK_SECONDS seconds sleep. Then it will have 0 running tasks.
       wait_zero_running_tasks "${SERVICE_NAME}"
     }
-    test_update_jobs_no_running_tasks() {
+    test_jobs_no_running_tasks() {
       local TEST_NAME="${1}"
       local SERVICE_NAME="${2}"
       reset_gantry_env "${SERVICE_NAME}"
@@ -217,7 +220,7 @@ Describe 'update-jobs'
     BeforeEach "test_start ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME} ${TASK_SECONDS}"
     AfterEach "common_cleanup ${TEST_NAME} ${IMAGE_WITH_TAG} ${SERVICE_NAME}"
     It 'run_test'
-      When run test_update_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}"
+      When run test_jobs_no_running_tasks "${TEST_NAME}" "${SERVICE_NAME}"
       The status should be success
       The stdout should satisfy display_output
       The stdout should satisfy spec_expect_no_message ".+"
@@ -247,6 +250,7 @@ Describe 'update-jobs'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'update-jobs'
diff --git a/tests/gantry_login_docker_config_spec.sh b/tests/gantry_login_docker_config_spec.sh
@@ -103,6 +103,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_docker_config_default_config" "container_test:true" "coverage:true"
@@ -173,6 +174,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_no_message "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_message    "${REMOVED_IMAGE}.*${IMAGE_WITH_TAG}"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*${IMAGE_WITH_TAG}"
+      The stderr should satisfy spec_expect_message    "${DONE_REMOVING_IMAGES}"
     End
   End
   Describe "test_login_docker_config_label_override" "container_test:false" "coverage:true"
@@ -291,6 +293,7 @@ Describe 'login_docker_config'
       The stderr should satisfy spec_expect_message    "${SKIP_REMOVING_IMAGES}"
       The stderr should satisfy spec_expect_no_message "${REMOVED_IMAGE}.*"
       The stderr should satisfy spec_expect_no_message "${FAILED_TO_REMOVE_IMAGE}.*"
+      The stderr should satisfy spec_expect_no_message "${DONE_REMOVING_IMAGES}"
     End
   End
 End # Describe 'login_docker_config'