Skip to content

Commit

Permalink
Adds the ability to run a specific type of scenario multiple times
Browse files Browse the repository at this point in the history 
With the current implementation, all the scenarios of specific type
(for example, pod scenario) has to be executed together. All
pod_scenarios are followed by node_scenarios and so on.
(pod_scenarios -> node_scenarios -> pod_scenarios is not possible)
This commit enables the user to run a specific type of scenario
multiple times. For example, few pod_scenarios followed by
node_scenarios followed by few_scenarios.
  • Loading branch information
@yashashreesuresh @chaitanyaenr
yashashreesuresh authored and chaitanyaenr committed Oct 30, 2020
1 parent 6f31519 commit 47847d8
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 76 deletions.
23 changes: 13 additions & 10 deletions config/config.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
kraken:
kubeconfig_path: /root/.kube/config # Path to kubeconfig
exit_on_failure: False # Exit when a post action scenario fails
scenarios: # List of policies/chaos scenarios to load
- - scenarios/etcd.yml
- - scenarios/openshift-kube-apiserver.yml
- - scenarios/openshift-apiserver.yml
- - scenarios/regex_openshift_pod_kill.yml
- scenarios/post_action_regex.py
node_scenarios: # List of chaos node scenarios to load
- scenarios/node_scenarios_example.yml
time_scenarios: # List of chaos time scenarios to load
- scenarios/time_scenarios_example.yml
chaos_scenarios: # List of policies/chaos scenarios to load
- pod_scenarios: # List of chaos pod scenarios to load
- - scenarios/etcd.yml
- - scenarios/regex_openshift_pod_kill.yml
- scenarios/post_action_regex.py
- node_scenarios: # List of chaos node scenarios to load
- scenarios/node_scenarios_example.yml
- pod_scenarios:
- - scenarios/openshift-apiserver.yml
- - scenarios/openshift-kube-apiserver.yml
- time_scenarios: # List of chaos time scenarios to load
- scenarios/time_scenarios_example.yml

cerberus:
cerberus_enabled: False # Enable it when cerberus is previously installed
cerberus_url: # When cerberus_enabled is set to True, provide the url where cerberus publishes go/no-go signal
Expand Down
20 changes: 14 additions & 6 deletions docs/config.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,19 @@ Set the scenarios to inject and the tunings like duration to wait between each s
```
kraken:
kubeconfig_path: /root/.kube/config # Path to kubeconfig
scenarios: # List of policies/chaos scenarios to load
- scenarios/etcd.yml
- scenarios/openshift-kube-apiserver.yml
- scenarios/openshift-apiserver.yml
node_scenarios: # List of chaos node scenarios to load
- scenarios/node_scenarios_example.yml
exit_on_failure: False # Exit when a post action scenario fails
chaos_scenarios: # List of policies/chaos scenarios to load
- pod_scenarios: # List of chaos pod scenarios to load
- - scenarios/etcd.yml
- - scenarios/regex_openshift_pod_kill.yml
- scenarios/post_action_regex.py
- node_scenarios: # List of chaos node scenarios to load
- scenarios/node_scenarios_example.yml
- pod_scenarios:
- - scenarios/openshift-apiserver.yml
- - scenarios/openshift-kube-apiserver.yml
- time_scenarios: # List of chaos time scenarios to load
- scenarios/time_scenarios_example.yml
cerberus:
cerberus_enabled: False # Enable it when cerberus is previously installed
Expand All @@ -19,3 +26,4 @@ tunings:
wait_duration: 60 # Duration to wait between each chaos scenario
iterations: 1 # Number of times to execute the scenarios
daemon_mode: False # Iterations are set to infinity which means that the kraken will cause chaos forever
```
134 changes: 74 additions & 60 deletions run_kraken.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,61 @@ def post_actions(kubeconfig_path, scenario, failed_post_scenarios, pre_action_ou
return failed_post_scenarios


def pod_scenarios(scenarios_list, config, failed_post_scenarios):
try:
# Loop to run the scenarios starts here
for pod_scenario in scenarios_list:
if len(pod_scenario) > 1:
pre_action_output = run_post_action(kubeconfig_path, pod_scenario[1])
else:
pre_action_output = ''
runcommand.invoke("powerfulseal autonomous --use-pod-delete-instead-of-ssh-kill"
" --policy-file %s --kubeconfig %s --no-cloud"
" --inventory-kubernetes --headless"
% (pod_scenario[0], kubeconfig_path))

logging.info("Scenario: %s has been successfully injected!" % (pod_scenario[0]))
logging.info("Waiting for the specified duration: %s" % (wait_duration))
time.sleep(wait_duration)

failed_post_scenarios = post_actions(kubeconfig_path, pod_scenario,
failed_post_scenarios, pre_action_output)
publish_kraken_status(config, failed_post_scenarios)
except Exception as e:
logging.error("Failed to run scenario: %s. Encountered the following "
"exception: %s" % (pod_scenario[0], e))
return failed_post_scenarios


def node_scenarios(scenarios_list, config):
for node_scenario_config in scenarios_list:
with open(node_scenario_config, 'r') as f:
node_scenario_config = yaml.full_load(f)
for node_scenario in node_scenario_config['node_scenarios']:
node_scenario_object = get_node_scenario_object(node_scenario)
if node_scenario['actions']:
for action in node_scenario['actions']:
inject_node_scenario(action, node_scenario, node_scenario_object)
logging.info("Waiting for the specified duration: %s" % (wait_duration))
time.sleep(wait_duration)
cerberus_integration(config)
logging.info("")


def time_scenarios(scenarios_list, config):
for time_scenario_config in scenarios_list:
with open(time_scenario_config, 'r') as f:
scenario_config = yaml.full_load(f)
for time_scenario in scenario_config['time_scenarios']:
object_type, object_names = time_actions.skew_time(time_scenario)
not_reset = time_actions.check_date_time(object_type, object_names)
if len(not_reset) > 0:
logging.info('Object times were not reset')
logging.info("Waiting for the specified duration: %s" % (wait_duration))
time.sleep(wait_duration)
publish_kraken_status(config, not_reset)


# Main function
def main(cfg):
# Start kraken
Expand All @@ -162,10 +217,9 @@ def main(cfg):
if os.path.isfile(cfg):
with open(cfg, 'r') as f:
config = yaml.full_load(f)
global kubeconfig_path, wait_duration
kubeconfig_path = config["kraken"].get("kubeconfig_path", "")
scenarios = config["kraken"].get("scenarios", [])
node_scenarios = config["kraken"].get("node_scenarios", [])
time_scenarios = config['kraken'].get("time_scenarios", [])
chaos_scenarios = config["kraken"].get("chaos_scenarios", [])
wait_duration = config["tunings"].get("wait_duration", 60)
iterations = config["tunings"].get("iterations", 1)
daemon_mode = config["tunings"].get("daemon_mode", False)
Expand Down Expand Up @@ -205,63 +259,23 @@ def main(cfg):
while (int(iteration) < iterations):
# Inject chaos scenarios specified in the config
logging.info("Executing scenarios for iteration " + str(iteration))
if scenarios:
try:
# Loop to run the scenarios starts here
for scenario in scenarios:

if len(scenario) > 1:
pre_action_output = run_post_action(kubeconfig_path, scenario[1])
else:
pre_action_output = ''
runcommand.invoke("powerfulseal autonomous --use-pod-delete-instead-of-ssh-kill" # noqa
" --policy-file %s --kubeconfig %s --no-cloud"
" --inventory-kubernetes --headless"
% (scenario[0], kubeconfig_path))

logging.info("Scenario: %s has been successfully injected!" % (scenario[0]))
logging.info("Waiting for the specified duration: %s" % (wait_duration))
time.sleep(wait_duration)

failed_post_scenarios = post_actions(kubeconfig_path, scenario,
failed_post_scenarios,
pre_action_output)
publish_kraken_status(config, failed_post_scenarios)
except Exception as e:
logging.error("Failed to run scenario: %s. Encountered the following "
"exception: %s" % (scenario[0], e))

# Inject node chaos scenarios specified in the config
if node_scenarios:
for node_scenario_config in node_scenarios:
with open(node_scenario_config, 'r') as f:
node_scenario_config = yaml.full_load(f)
for node_scenario in node_scenario_config['node_scenarios']:
node_scenario_object = get_node_scenario_object(node_scenario)
if node_scenario['actions']:
for action in node_scenario['actions']:
inject_node_scenario(action, node_scenario,
node_scenario_object)
logging.info("Waiting for the specified duration: %s"
% (wait_duration))
time.sleep(wait_duration)
cerberus_integration(config)
logging.info("")

# Inject time skew chaos scenarios specified in the config
if time_scenarios:
for time_scenario_config in time_scenarios:
with open(time_scenario_config, 'r') as f:
scenario_config = yaml.full_load(f)
for time_scenario in scenario_config['time_scenarios']:
object_type, object_names = time_actions.skew_time(time_scenario)
not_reset = time_actions.check_date_time(object_type, object_names)
if len(not_reset) > 0:
logging.info('Object times were not reset')
logging.info("Waiting for the specified duration: %s"
% wait_duration)
time.sleep(wait_duration)
publish_kraken_status(config, not_reset)
if chaos_scenarios:
for scenario in chaos_scenarios:
scenario_type = list(scenario.keys())[0]
scenarios_list = scenario[scenario_type]
if scenarios_list:
# Inject pod chaos scenarios specified in the config
if scenario_type == "pod_scenarios":
failed_post_scenarios = pod_scenarios(scenarios_list, config,
failed_post_scenarios)

# Inject node chaos scenarios specified in the config
elif scenario_type == "node_scenarios":
node_scenarios(scenarios_list, config)

# Inject time skew chaos scenarios specified in the config
elif scenario_type == "time_scenarios":
time_scenarios(scenarios_list, config)

iteration += 1
logging.info("")
Expand Down

0 comments on commit 47847d8

Please sign in to comment.