Split spec files to examples programmatically

lib/rspecq/worker.rb

@@ -172,9 +172,39 @@ def try_publish_queue!(queue)
       puts "Published queue (size=#{queue.publish(jobs, fail_fast)})"
     end
 
+    # Split a list of spec files into their individual examples.
+    #
+    # NOTE: RSpec has to load the files before we can split them as individual
+    # examples. In case a file to be splitted fails to be loaded
+    # (e.g. contains a syntax error), we return the files unchanged, thereby
+    # falling back to scheduling them as whole files. Their errors will be
+    # reported in the normal flow when they're eventually picked up by a worker.
+    def files_to_example_ids(files)
+      out = StringIO.new
+
+      reset_rspec_state!
+
+      dry_run_before = RSpec.configuration.dry_run
+
+      RSpec.configuration.add_formatter(RSpec::Core::Formatters::JsonFormatter.new(out))
+      RSpec.configuration.files_or_directories_to_run = files_or_dirs_to_run
+      RSpec.configuration.dry_run = true
+
+      opts = RSpec::Core::ConfigurationOptions.new(files)
+      result = RSpec::Core::Runner.new(opts).run($stderr, $stdout)
+
+      return files if result != 0
+
+      JSON.parse(out.string)["examples"].map { |e| e["id"] }
+    ensure
+      RSpec.configuration.dry_run = dry_run_before
+      reset_rspec_state!
+    end
+
     private
 
     def reset_rspec_state!
+      RSpec.reset
       RSpec.clear_examples
 
       # see https://github.com/rspec/rspec-core/pull/2723
@@ -194,38 +224,6 @@ def reset_rspec_state!
       RSpec.world.wants_to_quit = false
     end
 
-    # NOTE: RSpec has to load the files before we can split them as individual
-    # examples. In case a file to be splitted fails to be loaded
-    # (e.g. contains a syntax error), we return the files unchanged, thereby
-    # falling back to scheduling them as whole files. Their errors will be
-    # reported in the normal flow when they're eventually picked up by a worker.
-    def files_to_example_ids(files)
-      cmd = "DISABLE_SPRING=1 bundle exec rspec --dry-run --format json #{files.join(' ')}"
-      out, err, cmd_result = Open3.capture3(cmd)
-
-      if !cmd_result.success?
-        rspec_output = begin
-          JSON.parse(out)
-        rescue JSON::ParserError
-          out
-        end
-
-        log_event(
-          "Failed to split slow files, falling back to regular scheduling.\n #{err}",
-          "error",
-          rspec_stdout: rspec_output,
-          rspec_stderr: err,
-          cmd_result: cmd_result.inspect
-        )
-
-        pp rspec_output
-
-        return files
-      end
-
-      JSON.parse(out)["examples"].map { |e| e["id"] }
-    end
-
     def relative_path(job)
       @cwd ||= Pathname.new(Dir.pwd)
       "./#{Pathname.new(job).relative_path_from(@cwd)}"

test/sample_suites/files_to_examples/spec/bar_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe "baz" do
+  it { expect(1).to eq 2 }
+end

test/sample_suites/files_to_examples/spec/foo_spec.rb

@@ -0,0 +1,15 @@
+RSpec.describe "foo" do
+  it "works" do
+    expect(true).to be true
+  end
+
+  context "hmm" do
+    it "or not" do
+      expect(true).to be true
+    end
+
+    it "well, maybe" do
+      expect(true).to be true
+    end
+  end
+end

test/sample_suites/files_to_examples_fallback/spec/bar_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe "baz" do
+  it { expect(1).to eq 2 }
+end

test/sample_suites/files_to_examples_fallback/spec/foo_spec.rb

@@ -0,0 +1,15 @@
+IdontExistz1337.describe "foo" do
+  it "works" do
+    expect(true).to be true
+  end
+
+  context "hmm" do
+    it "or not" do
+      expect(true).to be true
+    end
+
+    it "well, maybe" do
+      expect(true).to be true
+    end
+  end
+end

test/test_helpers.rb

@@ -58,11 +58,14 @@ def silent
     end
 
     begin
-      orig = $stdout.clone
+      orig_out = $stdout.clone
+      orig_err = $stderr.clone
       $stdout.reopen(File::NULL, "w")
+      $stderr.reopen(File::NULL, "w")
       yield
     ensure
-      $stdout.reopen(orig)
+      $stdout.reopen(orig_out)
+      $stderr.reopen(orig_err)
     end
   end
 end

test/test_worker.rb

@@ -0,0 +1,41 @@
+require "test_helpers"
+
+class TestWorker < RSpecQTest
+  def test_files_to_example_ids
+    worker = new_worker("files_to_examples")
+
+    expected = [
+      "./test/sample_suites/files_to_examples/spec/foo_spec.rb[1:1]",
+      "./test/sample_suites/files_to_examples/spec/foo_spec.rb[1:2:1]",
+      "./test/sample_suites/files_to_examples/spec/foo_spec.rb[1:2:2]",
+      "./test/sample_suites/files_to_examples/spec/bar_spec.rb[1:1]",
+    ].sort
+
+    actual = worker.files_to_example_ids(
+      [
+        "./test/sample_suites/files_to_examples/spec/foo_spec.rb",
+        "./test/sample_suites/files_to_examples/spec/bar_spec.rb",
+      ]
+    ).sort
+
+    assert_equal expected, actual
+  end
+
+  def test_files_to_example_ids_failure_fallback
+    worker = new_worker("files_to_examples_fallback")
+
+    expected = [
+      "./test/sample_suites/files_to_examples_fallback/spec/foo_spec.rb",
+      "./test/sample_suites/files_to_examples_fallback/spec/bar_spec.rb"
+    ].sort
+
+    actual = worker.files_to_example_ids(
+      [
+        "./test/sample_suites/files_to_examples_fallback/spec/foo_spec.rb",
+        "./test/sample_suites/files_to_examples_fallback/spec/bar_spec.rb",
+      ]
+    ).sort
+
+    assert_equal expected, actual
+  end
+end