diff --git a/docs/node_scenarios.md b/docs/node_scenarios.md index 0fe503c5..3913c0f1 100644 --- a/docs/node_scenarios.md +++ b/docs/node_scenarios.md @@ -9,8 +9,9 @@ The following node chaos scenarios are supported: 5. **node_reboot_scenario**: Scenario to reboot the node instance. 6. **stop_kubelet_scenario**: Scenario to stop the kubelet of the node instance. 7. **stop_start_kubelet_scenario**: Scenario to stop and start the kubelet of the node instance. -8. **node_crash_scenario**: Scenario to crash the node instance. -9. **stop_start_helper_node_scenario**: Scenario to stop and start the helper node and check service status. +8. **restart_kubelet_scenario**: Scenario to restart the kubelet of the node instance. +9. **node_crash_scenario**: Scenario to crash the node instance. +10. **stop_start_helper_node_scenario**: Scenario to stop and start the helper node and check service status. **NOTE**: If the node does not recover from the node_crash_scenario injection, reboot the node to get it back to Ready state. diff --git a/kraken/node_actions/abstract_node_scenarios.py b/kraken/node_actions/abstract_node_scenarios.py index d133a67d..73928375 100644 --- a/kraken/node_actions/abstract_node_scenarios.py +++ b/kraken/node_actions/abstract_node_scenarios.py @@ -65,6 +65,26 @@ def stop_start_kubelet_scenario(self, instance_kill_count, node, timeout): self.node_reboot_scenario(instance_kill_count, node, timeout) logging.info("stop_start_kubelet_scenario has been successfully injected!") + + # Node scenario to restart the kubelet + def restart_kubelet_scenario(self, instance_kill_count, node, timeout): + for _ in range(instance_kill_count): + try: + logging.info("Starting restart_kubelet_scenario injection") + logging.info("Restarting the kubelet of the node %s" % (node)) + runcommand.run("oc debug node/" + node + " -- chroot /host systemctl restart kubelet &") + nodeaction.wait_for_not_ready_status(node, timeout, self.kubecli) + nodeaction.wait_for_ready_status(node, timeout, self.kubecli) + logging.info("The kubelet of the node %s has been restarted" % (node)) + logging.info("restart_kubelet_scenario has been successfuly injected!") + except Exception as e: + logging.error( + "Failed to restart the kubelet of the node. Encountered following " "exception: %s. Test Failed" % (e) + ) + logging.error("restart_kubelet_scenario injection failed!") + sys.exit(1) + + # Node scenario to crash the node def node_crash_scenario(self, instance_kill_count, node, timeout): for _ in range(instance_kill_count): diff --git a/kraken/node_actions/run.py b/kraken/node_actions/run.py index ed5b0289..42771ac3 100644 --- a/kraken/node_actions/run.py +++ b/kraken/node_actions/run.py @@ -130,6 +130,8 @@ def inject_node_scenario(action, node_scenario, node_scenario_object, kubecli: K node_scenario_object.node_reboot_scenario(run_kill_count, single_node, timeout) elif action == "stop_start_kubelet_scenario": node_scenario_object.stop_start_kubelet_scenario(run_kill_count, single_node, timeout) + elif action == "restart_kubelet_scenario": + node_scenario_object.restart_kubelet_scenario(run_kill_count, single_node, timeout) elif action == "stop_kubelet_scenario": node_scenario_object.stop_kubelet_scenario(run_kill_count, single_node, timeout) elif action == "node_crash_scenario":