stop workers during online backup

this ensures more consistent on-disk data

definitions/checks/pulpcore/no_running_tasks.rb

@@ -0,0 +1,22 @@
+module Checks::Pulpcore
+  class NoRunningTasks < ForemanMaintain::Check
+    metadata do
+      for_feature :pulpcore
+      description 'Check for running pulpcore tasks'
+      tags :pre_upgrade
+    end
+
+    def run
+      tasks = feature(:pulpcore).running_tasks
+      assert(tasks.empty?, failure_message(tasks.length),
+        :next_steps => [Procedures::Pulpcore::WaitForTasks.new])
+    end
+
+    private
+
+    def failure_message(task_count)
+      "There are #{task_count} active task(s) in the system." \
+      "\nPlease wait for these to complete."
+    end
+  end
+end

definitions/features/dynflow_sidekiq.rb

@@ -20,6 +20,10 @@ def services
     end
   end
 
+  def workers
+    services.reject { |s| s.name.end_with?('@orchestrator') }
+  end
+
   private
 
   def instance_priority(instance)

definitions/features/pulpcore.rb

@@ -3,10 +3,34 @@
 class Features::Pulpcore < ForemanMaintain::Feature
   include ForemanMaintain::Concerns::PulpCommon
 
+  TIMEOUT_FOR_TASKS_STATUS = 300
+  RETRY_INTERVAL_FOR_TASKS_STATE = 10
+
   metadata do
     label :pulpcore
   end
 
+  def cli(args)
+    parse_json(execute("pulp --format json #{args}"))
+  end
+
+  def running_tasks
+    cli('task list --state-in running --state-in canceling')
+  end
+
+  def wait_for_tasks(spinner)
+    Timeout.timeout(TIMEOUT_FOR_TASKS_STATUS) do
+      while (task_count = running_tasks.length) != 0
+        puts "\nThere are #{task_count} tasks."
+        spinner.update "Waiting #{RETRY_INTERVAL_FOR_TASKS_STATE} seconds before retry."
+        sleep RETRY_INTERVAL_FOR_TASKS_STATE
+      end
+    end
+  rescue Timeout::Error => e
+    logger.error e.message
+    puts "\nTimeout: #{e.message}. Try again."
+  end
+
   def services
     redis_services = feature(:redis) ? feature(:redis).services : []
 

definitions/procedures/pulpcore/wait_for_tasks.rb

@@ -0,0 +1,15 @@
+module Procedures::Pulpcore
+  class WaitForTasks < ForemanMaintain::Procedure
+    metadata do
+      for_feature :pulpcore
+      description 'Fetch tasks status and wait till they finish'
+      advanced_run false
+    end
+
+    def run
+      with_spinner("waiting for tasks to finish") do |spinner|
+        feature(:foreman_tasks).wait_for_tasks(spinner)
+      end
+    end
+  end
+end

definitions/scenarios/backup.rb

@@ -1,5 +1,20 @@
 module ForemanMaintain::Scenarios
-  class Backup < ForemanMaintain::Scenario
+  class BackupBase < ForemanMaintain::Scenario
+    private
+
+    def strategy
+      context.get(:strategy)
+    end
+
+    def online_worker_services
+      services = []
+      services += feature(:dynflow_sidekiq).workers if feature(:dynflow_sidekiq)
+      services += feature(:pulpcore).configured_workers if feature(:pulpcore)
+      services
+    end
+  end
+
+  class Backup < BackupBase
     metadata do
       description 'Backup'
       manual_detection
@@ -18,6 +33,8 @@ class Backup < ForemanMaintain::Scenario
 
     def compose
       check_valid_strategy
+      add_step(Checks::ForemanTasks::NotRunning)
+      add_step(Checks::Pulpcore::NoRunningTasks)
       safety_confirmation
       add_step_with_context(Procedures::Backup::AccessibilityConfirmation) if strategy == :offline
       add_step_with_context(Procedures::Backup::PrepareDirectory)
@@ -106,6 +123,9 @@ def include_dumps
     end
 
     def add_online_backup_steps
+      services = online_worker_services
+      add_step(Procedures::Service::Stop.new(:only => services)) unless services.empty?
+
       add_step_with_context(Procedures::Backup::ConfigFiles, :ignore_changed_files => true,
         :online_backup => true)
       add_step_with_context(Procedures::Backup::Pulp, :ensure_unchanged => true)
@@ -114,18 +134,16 @@ def add_online_backup_steps
         Procedures::Backup::Online::ForemanDB,
         Procedures::Backup::Online::PulpcoreDB
       )
-    end
 
-    def strategy
-      context.get(:strategy)
+      add_step(Procedures::Service::Start.new(:only => services)) unless services.empty?
     end
 
     def include_db_dumps?
       !!context.get(:include_db_dumps)
     end
   end
 
-  class BackupRescueCleanup < ForemanMaintain::Scenario
+  class BackupRescueCleanup < BackupBase
     metadata do
       description 'Failed backup cleanup'
       manual_detection
@@ -140,6 +158,10 @@ def compose
         add_step_with_context(Procedures::Service::Start)
         add_steps_with_context(find_procedures(:maintenance_mode_off))
       end
+      if strategy == :online
+        services = online_worker_services
+        add_step(Procedures::Service::Start.new(:only => services)) unless services.empty?
+      end
       add_step_with_context(Procedures::Backup::Clean)
     end
 
@@ -149,11 +171,5 @@ def set_context_mapping
       context.map(:preserve_dir,
         Procedures::Backup::Clean => :preserve_dir)
     end
-
-    private
-
-    def strategy
-      context.get(:strategy)
-    end
   end
 end

test/definitions/scenarios/backup_test.rb

@@ -10,6 +10,22 @@ module Scenarios
           db.any_instance.stubs(:local?).returns(true)
         end
       end
+
+      assume_feature_present(:pulpcore) do |feature|
+        feature.any_instance.stubs(:configured_workers).returns([existing_pulpcore_worker])
+      end
+
+      assume_feature_present(:dynflow_sidekiq) do |feature|
+        feature.any_instance.stubs(:workers).returns([existing_dynflow_worker])
+      end
+    end
+
+    let(:existing_pulpcore_worker) { existing_system_service('pulpcore-worker@1', 10) }
+    let(:existing_dynflow_worker) { existing_system_service('dynflow-sidekiq@worker-1', 10) }
+
+    let(:checks) do
+      [Checks::ForemanTasks::NotRunning,
+       Checks::Pulpcore::NoRunningTasks]
     end
 
     describe 'offline' do
@@ -18,6 +34,9 @@ module Scenarios
       end
 
       it 'composes all steps' do
+        checks.each do |check|
+          assert_scenario_has_step(scenario, check)
+        end
         assert_scenario_has_step(scenario, Procedures::Backup::AccessibilityConfirmation)
         assert_scenario_has_step(scenario, Procedures::Backup::PrepareDirectory)
         assert_scenario_has_step(scenario, Procedures::Backup::Metadata)
@@ -42,17 +61,28 @@ module Scenarios
       end
 
       it 'composes all steps' do
+        checks.each do |check|
+          assert_scenario_has_step(scenario, check)
+        end
         assert_scenario_has_step(scenario, Procedures::Backup::Online::SafetyConfirmation)
         refute_scenario_has_step(scenario, Procedures::Backup::AccessibilityConfirmation)
         assert_scenario_has_step(scenario, Procedures::Backup::PrepareDirectory)
         assert_scenario_has_step(scenario, Procedures::Backup::Metadata)
-        refute_scenario_has_step(scenario, Procedures::Service::Stop)
+        assert_scenario_has_step(scenario, Procedures::Service::Stop) do |step|
+          assert_includes(step.common_options[:only], existing_pulpcore_worker)
+          assert_includes(step.common_options[:only], existing_dynflow_worker)
+          assert_equal(step.common_options[:only].length, 2)
+        end
         assert_scenario_has_step(scenario, Procedures::Backup::ConfigFiles)
         assert_scenario_has_step(scenario, Procedures::Backup::Pulp)
         assert_scenario_has_step(scenario, Procedures::Backup::Online::CandlepinDB)
         assert_scenario_has_step(scenario, Procedures::Backup::Online::ForemanDB)
         assert_scenario_has_step(scenario, Procedures::Backup::Online::PulpcoreDB)
-        refute_scenario_has_step(scenario, Procedures::Service::Start)
+        assert_scenario_has_step(scenario, Procedures::Service::Start) do |step|
+          assert_includes(step.common_options[:only], existing_pulpcore_worker)
+          assert_includes(step.common_options[:only], existing_dynflow_worker)
+          assert_equal(step.common_options[:only].length, 2)
+        end
         assert_scenario_has_step(scenario, Procedures::Backup::CompressData)
       end
     end
@@ -61,6 +91,19 @@ module Scenarios
   describe ForemanMaintain::Scenarios::BackupRescueCleanup do
     include DefinitionsTestHelper
 
+    before do
+      assume_feature_present(:pulpcore) do |feature|
+        feature.any_instance.stubs(:configured_workers).returns([existing_pulpcore_worker])
+      end
+
+      assume_feature_present(:dynflow_sidekiq) do |feature|
+        feature.any_instance.stubs(:workers).returns([existing_dynflow_worker])
+      end
+    end
+
+    let(:existing_pulpcore_worker) { existing_system_service('pulpcore-worker@1', 10) }
+    let(:existing_dynflow_worker) { existing_system_service('dynflow-sidekiq@worker-1', 10) }
+
     describe 'offline' do
       let(:scenario) do
         ForemanMaintain::Scenarios::BackupRescueCleanup.new(:backup_dir => '.',
@@ -80,7 +123,11 @@ module Scenarios
       end
 
       it 'composes all steps' do
-        refute_scenario_has_step(scenario, Procedures::Service::Start)
+        assert_scenario_has_step(scenario, Procedures::Service::Start) do |step|
+          assert_includes(step.common_options[:only], existing_pulpcore_worker)
+          assert_includes(step.common_options[:only], existing_dynflow_worker)
+          assert_equal(step.common_options[:only].length, 2)
+        end
         assert_scenario_has_step(scenario, Procedures::Backup::Clean)
       end
     end