Merge pull request #713 from cybozu-go/test-too-long-repair

Add test for too long repair execution
cybozu-go · Mar 27, 2024 · 9d615b6 · 9d615b6
2 parents 269dc52 + aeb4ae1
commit 9d615b6
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 1 deletion.
diff --git a/mtest/cke-cluster.yml b/mtest/cke-cluster.yml
@@ -25,6 +25,7 @@ repair:
     - operation: "op1"
       repair_steps:
       - repair_command: ["sh", "-c", "touch /tmp/mtest-repair-$1", "repair"]
+        command_timeout_seconds: 30
         need_drain: true
         watch_seconds: 30
       health_check_command: ["sh", "-c", "test -f /tmp/mtest-repair-$1 && echo true", "health_check"]

diff --git a/mtest/repair_test.go b/mtest/repair_test.go
@@ -128,6 +128,34 @@ func testRepairOperations() {
 		ckecliSafe("repair-queue", "delete-finished")
 		waitRepairEmpty(cluster)
 
+		By("setting non-returning repair command")
+		cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].RepairCommand = []string{"sh", "-c", "exec sleep infinity", "sleep-infinity"}
+		_, err = ckecliClusterSet(cluster)
+		Expect(err).NotTo(HaveOccurred())
+		time.Sleep(time.Second * 3)
+
+		repairQueueAdd(node1)
+		waitRepairFailure(cluster)
+
+		ckecliSafe("repair-queue", "delete-finished")
+		waitRepairEmpty(cluster)
+
+		By("setting non-returning repair command and long command timeout")
+		originalCommandTimeoutSeconds := cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].CommandTimeoutSeconds
+
+		longCommandTimeout := 90 // > (timeout of repairShouldNotProceed())
+		cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].CommandTimeoutSeconds = &longCommandTimeout
+		_, err = ckecliClusterSet(cluster)
+		Expect(err).NotTo(HaveOccurred())
+		time.Sleep(time.Second * 3)
+
+		repairQueueAdd(node1)
+		repairShouldNotProceed()
+
+		time.Sleep(time.Second * time.Duration(longCommandTimeout)) // wait for CKE to update the queue entry
+		ckecliSafe("repair-queue", "delete-finished")
+		waitRepairEmpty(cluster)
+
 		By("setting noop repair command")
 		cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].RepairCommand = []string{"true"}
 		_, err = ckecliClusterSet(cluster)
@@ -155,8 +183,9 @@ func testRepairOperations() {
 		ckecliSafe("repair-queue", "delete", strconv.Itoa(currentWriteIndex-1))
 		waitRepairEmpty(cluster)
 
-		By("restoring repair command and watch duration")
+		By("restoring repair command, command timeout, and watch duration")
 		cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].RepairCommand = originalRepairCommand
+		cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].CommandTimeoutSeconds = originalCommandTimeoutSeconds
 		cluster.Repair.RepairProcedures[0].RepairOperations[0].RepairSteps[0].WatchSeconds = originalWatchSeconds
 		_, err = ckecliClusterSet(cluster)
 		Expect(err).NotTo(HaveOccurred())