diff --git a/definitions/checks/foreman_tasks/not_running.rb b/definitions/checks/foreman_tasks/not_running.rb index cc093b69a..6cc7f6433 100644 --- a/definitions/checks/foreman_tasks/not_running.rb +++ b/definitions/checks/foreman_tasks/not_running.rb @@ -6,17 +6,17 @@ class NotRunning < ForemanMaintain::Check tags :pre_upgrade after :foreman_tasks_not_paused before :check_old_foreman_tasks + param :wait_for_tasks, + 'Wait for tasks to finish or fail directly', + :required => false, + :default => true end def run task_count = feature(:foreman_tasks).running_tasks_count assert(task_count == 0, failure_message(task_count), - :next_steps => - [Procedures::ForemanTasks::FetchTasksStatus.new(:state => 'running'), - Procedures::ForemanTasks::UiInvestigate.new( - 'search_query' => search_query_for_running_tasks - )]) + :next_steps => calculate_next_steps) end private @@ -30,5 +30,18 @@ def failure_message(task_count) "There are #{task_count} active task(s) in the system." \ "\nPlease wait for these to complete or cancel them from the Monitor tab." end + + def calculate_next_steps + steps = [] + if @wait_for_tasks + steps << Procedures::ForemanTasks::FetchTasksStatus.new(:state => 'running') + unless assumeyes? + steps << Procedures::ForemanTasks::UiInvestigate.new( + 'search_query' => search_query_for_running_tasks + ) + end + end + steps + end end end diff --git a/definitions/checks/pulpcore/no_running_tasks.rb b/definitions/checks/pulpcore/no_running_tasks.rb new file mode 100644 index 000000000..3c6ee1c54 --- /dev/null +++ b/definitions/checks/pulpcore/no_running_tasks.rb @@ -0,0 +1,32 @@ +module Checks::Pulpcore + class NoRunningTasks < ForemanMaintain::Check + metadata do + for_feature :pulpcore + description 'Check for running pulpcore tasks' + tags :pre_upgrade + param :wait_for_tasks, + 'Wait for tasks to finish or fail directly', + :required => false + end + + def run + tasks = feature(:pulpcore).running_tasks + assert( + tasks.empty?, + failure_message(tasks.length), + :next_steps => calculate_next_steps + ) + end + + private + + def failure_message(task_count) + "There are #{task_count} active task(s) in the system." \ + "\nPlease wait for these to complete." + end + + def calculate_next_steps + @wait_for_tasks ? [Procedures::Pulpcore::WaitForTasks.new] : [] + end + end +end diff --git a/definitions/features/dynflow_sidekiq.rb b/definitions/features/dynflow_sidekiq.rb index ba4e3e302..68290640d 100644 --- a/definitions/features/dynflow_sidekiq.rb +++ b/definitions/features/dynflow_sidekiq.rb @@ -20,6 +20,10 @@ def services end end + def workers + services.reject { |service| service.name.end_with?('@orchestrator') } + end + private def instance_priority(instance) diff --git a/definitions/features/pulpcore.rb b/definitions/features/pulpcore.rb index 111fe3718..e40165680 100644 --- a/definitions/features/pulpcore.rb +++ b/definitions/features/pulpcore.rb @@ -3,10 +3,34 @@ class Features::Pulpcore < ForemanMaintain::Feature include ForemanMaintain::Concerns::PulpCommon + TIMEOUT_FOR_TASKS_STATUS = 300 + RETRY_INTERVAL_FOR_TASKS_STATE = 10 + metadata do label :pulpcore end + def cli(args) + parse_json(execute("pulp --format json #{args}")) + end + + def running_tasks + cli('task list --state-in running --state-in canceling') + end + + def wait_for_tasks(spinner, timeout_for_tasks_status = TIMEOUT_FOR_TASKS_STATUS) + Timeout.timeout(timeout_for_tasks_status) do + while (task_count = running_tasks.length) != 0 + puts "\nThere are #{task_count} tasks." + spinner.update "Waiting #{RETRY_INTERVAL_FOR_TASKS_STATE} seconds before retry." + sleep RETRY_INTERVAL_FOR_TASKS_STATE + end + end + rescue Timeout::Error => e + logger.error(e.message) + puts "\nTimeout: #{e.message}. Try again." + end + def services redis_services = feature(:redis) ? feature(:redis).services : [] diff --git a/definitions/procedures/pulpcore/wait_for_tasks.rb b/definitions/procedures/pulpcore/wait_for_tasks.rb new file mode 100644 index 000000000..9128edc5f --- /dev/null +++ b/definitions/procedures/pulpcore/wait_for_tasks.rb @@ -0,0 +1,15 @@ +module Procedures::Pulpcore + class WaitForTasks < ForemanMaintain::Procedure + metadata do + for_feature :pulpcore + description 'Fetch tasks status and wait till they finish' + advanced_run false + end + + def run + with_spinner("waiting for tasks to finish") do |spinner| + feature(:pulpcore).wait_for_tasks(spinner) + end + end + end +end diff --git a/definitions/scenarios/backup.rb b/definitions/scenarios/backup.rb index 82cc16e7a..046b47838 100644 --- a/definitions/scenarios/backup.rb +++ b/definitions/scenarios/backup.rb @@ -1,5 +1,24 @@ module ForemanMaintain::Scenarios - class Backup < ForemanMaintain::Scenario + class BackupBase < ForemanMaintain::Scenario + private + + def strategy + context.get(:strategy) + end + + def wait_for_tasks + !!context.get(:wait_for_tasks) + end + + def online_worker_services + services = [] + services += feature(:dynflow_sidekiq).workers if feature(:dynflow_sidekiq) + services += feature(:pulpcore).configured_workers if feature(:pulpcore) + services + end + end + + class Backup < BackupBase metadata do description 'Backup' manual_detection @@ -14,10 +33,13 @@ class Backup < ForemanMaintain::Scenario param :proxy_features, 'List of proxy features to backup (default: all)', :array => true param :skip_pulp_content, 'Skip Pulp content during backup' param :tar_volume_size, 'Size of tar volume (indicates splitting)' + param :wait_for_tasks, 'Wait for running tasks to complete instead of aborting' end def compose check_valid_strategy + add_step(Checks::ForemanTasks::NotRunning.new(:wait_for_tasks => wait_for_tasks)) + add_step(Checks::Pulpcore::NoRunningTasks.new(:wait_for_tasks => wait_for_tasks)) safety_confirmation add_step_with_context(Procedures::Backup::AccessibilityConfirmation) if strategy == :offline add_step_with_context(Procedures::Backup::PrepareDirectory) @@ -104,6 +126,9 @@ def include_dumps end def add_online_backup_steps + services = online_worker_services + add_step(Procedures::Service::Stop.new(:only => services)) unless services.empty? + add_step_with_context(Procedures::Backup::ConfigFiles, :ignore_changed_files => true, :online_backup => true) add_step_with_context(Procedures::Backup::Pulp, :ensure_unchanged => true) @@ -112,10 +137,8 @@ def add_online_backup_steps Procedures::Backup::Online::ForemanDB, Procedures::Backup::Online::PulpcoreDB ) - end - def strategy - context.get(:strategy) + add_step(Procedures::Service::Start.new(:only => services)) unless services.empty? end def include_db_dumps? @@ -123,7 +146,7 @@ def include_db_dumps? end end - class BackupRescueCleanup < ForemanMaintain::Scenario + class BackupRescueCleanup < BackupBase metadata do description 'Failed backup cleanup' manual_detection @@ -138,6 +161,10 @@ def compose add_step_with_context(Procedures::Service::Start) add_steps_with_context(find_procedures(:maintenance_mode_off)) end + if strategy == :online + services = online_worker_services + add_step(Procedures::Service::Start.new(:only => services)) unless services.empty? + end add_step_with_context(Procedures::Backup::Clean) end @@ -147,11 +174,5 @@ def set_context_mapping context.map(:preserve_dir, Procedures::Backup::Clean => :preserve_dir) end - - private - - def strategy - context.get(:strategy) - end end end diff --git a/lib/foreman_maintain/cli/backup_command.rb b/lib/foreman_maintain/cli/backup_command.rb index 0b4c93180..fa4a7859e 100644 --- a/lib/foreman_maintain/cli/backup_command.rb +++ b/lib/foreman_maintain/cli/backup_command.rb @@ -57,6 +57,7 @@ def common_backup_options option '--features', 'FEATURES', "#{proxy_name} features to include in the backup. " \ 'Valid features are tftp, dns, dhcp, openscap, and all.', :multivalued => true + option '--wait-for-tasks', :flag, 'Wait for running tasks to complete instead of aborting' end # rubocop:enable Metrics/MethodLength @@ -89,6 +90,7 @@ def backup_scenario(options, strategy) :tar_volume_size => split_pulp_tar, :skip_pulp_content => skip_pulp_content?, :incremental_dir => incremental, + :wait_for_tasks => wait_for_tasks?, }.merge(options)) end diff --git a/test/definitions/checks/foreman_tasks/not_running_test.rb b/test/definitions/checks/foreman_tasks/not_running_test.rb index a308f02f9..3db35fa08 100644 --- a/test/definitions/checks/foreman_tasks/not_running_test.rb +++ b/test/definitions/checks/foreman_tasks/not_running_test.rb @@ -3,25 +3,49 @@ describe Checks::ForemanTasks::NotRunning do include DefinitionsTestHelper - subject do - Checks::ForemanTasks::NotRunning.new - end + context 'with default params' do + subject do + Checks::ForemanTasks::NotRunning.new + end + + it 'passes when not active tasks are present' do + assume_feature_present(:foreman_tasks, :running_tasks_count => 0) + result = run_check(subject) + assert result.success?, 'Check expected to succeed' + end - it 'passes when not active tasks are present' do - assume_feature_present(:foreman_tasks, :running_tasks_count => 0) - result = run_check(subject) - assert result.success?, 'Check expected to succeed' + it 'fails when running/paused tasks are present' do + assume_feature_present(:foreman_tasks, :running_tasks_count => 5) + result = run_check(subject) + assert result.fail?, 'Check expected to fail' + msg = 'There are 5 active task(s) in the system.' + msg += "\nPlease wait for these to complete or cancel them from the Monitor tab." + assert_match msg, result.output + assert_equal [Procedures::ForemanTasks::FetchTasksStatus, + Procedures::ForemanTasks::UiInvestigate], + subject.next_steps.map(&:class) + end end - it 'fails when running/paused tasks are present' do - assume_feature_present(:foreman_tasks, :running_tasks_count => 5) - result = run_check(subject) - assert result.fail?, 'Check expected to fail' - msg = 'There are 5 active task(s) in the system.' - msg += "\nPlease wait for these to complete or cancel them from the Monitor tab." - assert_match msg, result.output - assert_equal [Procedures::ForemanTasks::FetchTasksStatus, - Procedures::ForemanTasks::UiInvestigate], - subject.next_steps.map(&:class) + context 'with wait_for_tasks=>false' do + subject do + Checks::ForemanTasks::NotRunning.new(:wait_for_tasks => false) + end + + it 'passes when not active tasks are present' do + assume_feature_present(:foreman_tasks, :running_tasks_count => 0) + result = run_check(subject) + assert result.success?, 'Check expected to succeed' + end + + it 'fails when running/paused tasks are present' do + assume_feature_present(:foreman_tasks, :running_tasks_count => 5) + result = run_check(subject) + assert result.fail?, 'Check expected to fail' + msg = 'There are 5 active task(s) in the system.' + msg += "\nPlease wait for these to complete or cancel them from the Monitor tab." + assert_match msg, result.output + assert_empty subject.next_steps.map(&:class) + end end end diff --git a/test/definitions/checks/pulpcore/no_running_tasks_test.rb b/test/definitions/checks/pulpcore/no_running_tasks_test.rb new file mode 100644 index 000000000..8267f4779 --- /dev/null +++ b/test/definitions/checks/pulpcore/no_running_tasks_test.rb @@ -0,0 +1,50 @@ +require 'test_helper' + +describe Checks::Pulpcore::NoRunningTasks do + include DefinitionsTestHelper + + context 'with default params' do + subject do + Checks::Pulpcore::NoRunningTasks.new + end + + it 'passes when not active tasks are present' do + assume_feature_present(:pulpcore, :running_tasks => []) + result = run_check(subject) + assert result.success?, 'Check expected to succeed' + end + + it 'fails when running/paused tasks are present' do + assume_feature_present(:pulpcore, :running_tasks => ['a_task']) + result = run_check(subject) + assert result.fail?, 'Check expected to fail' + msg = 'There are 1 active task(s) in the system.' + msg += "\nPlease wait for these to complete." + assert_match msg, result.output + assert_empty subject.next_steps.map(&:class) + end + end + + context 'with wait_for_tasks=>true' do + subject do + Checks::Pulpcore::NoRunningTasks.new(:wait_for_tasks => true) + end + + it 'passes when not active tasks are present' do + assume_feature_present(:pulpcore, :running_tasks => []) + result = run_check(subject) + assert result.success?, 'Check expected to succeed' + end + + it 'fails when running/paused tasks are present' do + assume_feature_present(:pulpcore, :running_tasks => ['a_task']) + result = run_check(subject) + assert result.fail?, 'Check expected to fail' + msg = 'There are 1 active task(s) in the system.' + msg += "\nPlease wait for these to complete." + assert_match msg, result.output + assert_equal [Procedures::Pulpcore::WaitForTasks], + subject.next_steps.map(&:class) + end + end +end diff --git a/test/definitions/scenarios/backup_test.rb b/test/definitions/scenarios/backup_test.rb index 5952d53eb..0b1ff92ff 100644 --- a/test/definitions/scenarios/backup_test.rb +++ b/test/definitions/scenarios/backup_test.rb @@ -10,6 +10,22 @@ module Scenarios db.any_instance.stubs(:local?).returns(true) end end + + assume_feature_present(:pulpcore) do |feature| + feature.any_instance.stubs(:configured_workers).returns([existing_pulpcore_worker]) + end + + assume_feature_present(:dynflow_sidekiq) do |feature| + feature.any_instance.stubs(:workers).returns([existing_dynflow_worker]) + end + end + + let(:existing_pulpcore_worker) { existing_system_service('pulpcore-worker@1', 10) } + let(:existing_dynflow_worker) { existing_system_service('dynflow-sidekiq@worker-1', 10) } + + let(:checks) do + [Checks::ForemanTasks::NotRunning, + Checks::Pulpcore::NoRunningTasks] end describe 'offline' do @@ -18,6 +34,11 @@ module Scenarios end it 'composes all steps' do + checks.each do |check| + assert_scenario_has_step(scenario, check) do |step| + refute step.options['wait_for_tasks'] + end + end assert_scenario_has_step(scenario, Procedures::Backup::AccessibilityConfirmation) assert_scenario_has_step(scenario, Procedures::Backup::PrepareDirectory) assert_scenario_has_step(scenario, Procedures::Backup::Metadata) @@ -36,31 +57,87 @@ module Scenarios end end + describe 'offline with wait_for_tasks' do + let(:scenario) do + ForemanMaintain::Scenarios::Backup.new(:backup_dir => '.', :strategy => :offline, + :wait_for_tasks => true) + end + + it 'composes all steps' do + checks.each do |check| + assert_scenario_has_step(scenario, check) do |step| + assert step.options['wait_for_tasks'] + end + end + end + end + describe 'online' do let(:scenario) do ForemanMaintain::Scenarios::Backup.new(:backup_dir => '.', :strategy => :online) end it 'composes all steps' do + checks.each do |check| + assert_scenario_has_step(scenario, check) do |step| + refute step.options['wait_for_tasks'] + end + end assert_scenario_has_step(scenario, Procedures::Backup::Online::SafetyConfirmation) refute_scenario_has_step(scenario, Procedures::Backup::AccessibilityConfirmation) assert_scenario_has_step(scenario, Procedures::Backup::PrepareDirectory) assert_scenario_has_step(scenario, Procedures::Backup::Metadata) - refute_scenario_has_step(scenario, Procedures::Service::Stop) + assert_scenario_has_step(scenario, Procedures::Service::Stop) do |step| + assert_includes(step.common_options[:only], existing_pulpcore_worker) + assert_includes(step.common_options[:only], existing_dynflow_worker) + assert_equal(step.common_options[:only].length, 2) + end assert_scenario_has_step(scenario, Procedures::Backup::ConfigFiles) assert_scenario_has_step(scenario, Procedures::Backup::Pulp) assert_scenario_has_step(scenario, Procedures::Backup::Online::CandlepinDB) assert_scenario_has_step(scenario, Procedures::Backup::Online::ForemanDB) assert_scenario_has_step(scenario, Procedures::Backup::Online::PulpcoreDB) - refute_scenario_has_step(scenario, Procedures::Service::Start) + assert_scenario_has_step(scenario, Procedures::Service::Start) do |step| + assert_includes(step.common_options[:only], existing_pulpcore_worker) + assert_includes(step.common_options[:only], existing_dynflow_worker) + assert_equal(step.common_options[:only].length, 2) + end assert_scenario_has_step(scenario, Procedures::Backup::CompressData) end end + + describe 'online with wait_for_tasks' do + let(:scenario) do + ForemanMaintain::Scenarios::Backup.new(:backup_dir => '.', :strategy => :online, + :wait_for_tasks => true) + end + + it 'composes all steps' do + checks.each do |check| + assert_scenario_has_step(scenario, check) do |step| + assert step.options['wait_for_tasks'] + end + end + end + end end describe ForemanMaintain::Scenarios::BackupRescueCleanup do include DefinitionsTestHelper + before do + assume_feature_present(:pulpcore) do |feature| + feature.any_instance.stubs(:configured_workers).returns([existing_pulpcore_worker]) + end + + assume_feature_present(:dynflow_sidekiq) do |feature| + feature.any_instance.stubs(:workers).returns([existing_dynflow_worker]) + end + end + + let(:existing_pulpcore_worker) { existing_system_service('pulpcore-worker@1', 10) } + let(:existing_dynflow_worker) { existing_system_service('dynflow-sidekiq@worker-1', 10) } + describe 'offline' do let(:scenario) do ForemanMaintain::Scenarios::BackupRescueCleanup.new(:backup_dir => '.', @@ -80,7 +157,11 @@ module Scenarios end it 'composes all steps' do - refute_scenario_has_step(scenario, Procedures::Service::Start) + assert_scenario_has_step(scenario, Procedures::Service::Start) do |step| + assert_includes(step.common_options[:only], existing_pulpcore_worker) + assert_includes(step.common_options[:only], existing_dynflow_worker) + assert_equal(step.common_options[:only].length, 2) + end assert_scenario_has_step(scenario, Procedures::Backup::Clean) end end