Report worker crashes to summary command

ruby/lib/ci/queue/build_record.rb

@@ -36,6 +36,14 @@ def reset_stats(stat_names)
         stat_names.each { |s| stats.delete(s) }
       end
 
+      def report_worker_error(_); end
+
+      def reset_worker_error; end
+
+      def worker_errors
+        {}
+      end
+
       private
 
       attr_reader :stats

ruby/lib/ci/queue/redis/build_record.rb

@@ -17,6 +17,21 @@ def queue_exhausted?
           @queue.exhausted?
         end
 
+        def report_worker_error(error)
+          redis.pipelined do |pipeline|
+            pipeline.hset(key('worker-errors'), config.worker_id, error.message)
+            pipeline.expire(key('worker-errors'), config.redis_ttl)
+          end
+        end
+
+        def worker_errors
+          redis.hgetall(key('worker-errors'))
+        end
+
+        def reset_worker_error
+          redis.hdel(key('worker-errors'), config.worker_id)
+        end
+
         def failed_tests
           redis.hkeys(key('error-reports'))
         end

ruby/lib/ci/queue/redis/worker.rb

@@ -95,6 +95,10 @@ def build
           @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
+        def report_worker_error(error)
+          build.report_worker_error(error)
+        end
+
         def acknowledge(test)
           test_key = test.id
           raise_on_mismatching_test(test_key)

ruby/lib/minitest/queue.rb

@@ -268,8 +268,12 @@ def run_from_queue(reporter, *)
       exit!(41)
     rescue => error
       reopen_previous_step
+      queue.report_worker_error(error)
       puts red("This worker exited because of an uncaught application error:")
       puts red("#{error.class}: #{error.message}")
+      error.backtrace.each do |frame|
+        puts red(frame)
+      end
       exit!(42)
     end
   end

ruby/lib/minitest/queue/build_status_reporter.rb

@@ -36,11 +36,18 @@ def report
           puts ""
         end
 
+        build.worker_errors.to_a.sort.each do |worker_id, error|
+          puts yellow("Worker #{worker_id } crashed")
+          puts error
+          puts ""
+        end
+
         errors.empty?
       end
 
       def success?
-        build.error_reports.empty?
+        build.error_reports.empty? &&
+          build.worker_errors.empty?
       end
 
       def record(*)

ruby/lib/minitest/queue/runner.rb

@@ -345,6 +345,7 @@ def run_tests_in_fork(queue)
 
       def reset_counters
         queue.build.reset_stats(BuildStatusRecorder::COUNTERS)
+        queue.build.reset_worker_error
       end
 
       def populate_queue

ruby/test/integration/minitest_redis_test.rb

@@ -952,6 +952,22 @@ def test_application_error
       end
 
       assert_equal 42, $?.exitstatus
+
+      out, err = capture_subprocess_io do
+        system(
+          @exe, 'report',
+          '--queue', @redis_url,
+          '--build', '1',
+          '--timeout', '1',
+          '--heartbeat',
+          chdir: 'test/fixtures/',
+          )
+      end
+
+      assert_includes out, "Worker 1 crashed"
+      assert_includes out, "Some error in the test framework"
+
+      assert_equal 1, $?.exitstatus
     end
 
     private