Avoid to run integration tests while testing all supported plugin (#1…

…5511) Update the test_plugins pipeline script to execute only the unit tests. Use the vendored JRuby in every Ruby related duty, such as running `bundler` and `gem`. Temporarily comments plugins that has needs to be fixed and already fails on their Travis CI. Executes the testing of input tier1 plugins in VM instead of Buildkite agent.
elastic · Dec 11, 2023 · 241c032 · 241c032
1 parent 042e5d4
commit 241c032
Show file tree

Hide file tree

Showing 3 changed files with 118 additions and 43 deletions.
diff --git a/.buildkite/supported_plugins_test_pipeline.yml b/.buildkite/supported_plugins_test_pipeline.yml
@@ -10,16 +10,45 @@ steps:
   - group: "Tier1 plugins test group"
     key: "tier1_plugins"
     steps:
-      - label: "Test Tier1 inputs"
-        command: ./ci/test_plugins.sh -t tier1 -k input
+      - label: "Test input file plugin"
+        command: |
+          source .buildkite/scripts/common/vm-agent.sh
+          ./ci/test_plugins.sh -p logstash-input-file
+        # Workaround for https://github.com/elastic/ingest-dev/issues/2676
+        agents:
+          provider: gcp
+          imageProject: elastic-images-prod
+          image: family/platform-ingest-logstash-ubuntu-2204
+          machineType: "n2-standard-4"
+          diskSizeGb: 120
+      - label: "Test rest of Tier1 inputs"
+        command: |
+          source .buildkite/scripts/common/vm-agent.sh
+          ./ci/test_plugins.sh -p logstash-input-azure_event_hubs,logstash-input-beats,logstash-input-elasticsearch,logstash-input-generator,logstash-input-heartbeat,logstash-input-http,logstash-input-http_poller,logstash-input-redis,logstash-input-stdin,logstash-input-syslog,logstash-input-udp
+        # Workaround for https://github.com/elastic/ingest-dev/issues/2676
+        agents:
+          provider: gcp
+          imageProject: elastic-images-prod
+          image: family/platform-ingest-logstash-ubuntu-2204
+          machineType: "n2-standard-4"
+          diskSizeGb: 120
       - label: "Test Tier1 filters"
         command: ./ci/test_plugins.sh -t tier1 -k filter
       - label: "Test Tier1 codecs"
         command: ./ci/test_plugins.sh -t tier1 -k codec
       - label: "Test Tier1 outputs"
         command: ./ci/test_plugins.sh -t tier1 -k output
       - label: "Test Tier1 integrations"
-        command: ./ci/test_plugins.sh -t tier1 -k integration
+        command: |
+          source .buildkite/scripts/common/vm-agent.sh
+          ./ci/test_plugins.sh -t tier1 -k integration
+        # Workaround to avoid errors on chmod of files
+        agents:
+          provider: gcp
+          imageProject: elastic-images-prod
+          image: family/platform-ingest-logstash-ubuntu-2204
+          machineType: "n2-standard-4"
+          diskSizeGb: 120  
 
   - group: "Tier2 plugins test group"
     key: "tier2_plugins"

diff --git a/ci/test_plugins.rb b/ci/test_plugins.rb
@@ -1,20 +1,22 @@
 # encoding: utf-8
 
 # Test the
-# - build (rspec),
+# - build (rspec unit testing),
 # - packaging (gem build)
 # - and deploy (bin/logstash-plugin install)
 # of a plugins inside the current Logstash, using its JRuby
 # Usage example:
 # bin/ruby ci/test_supported_plugins.rb -p logstash-integration-jdbc
 # bin/ruby ci/test_supported_plugins.rb -t tier1 -k input,codec,integration
+# bin/ruby ci/test_supported_plugins.rb -t tier1 -k input,codec,integration --split 1/3
 #
 # The script uses OS's temp folder unless the environment variable LOGSTASH_PLUGINS_TMP is specified.
 # The path of such variable should be absolute.
 
 require "open3"
 require "set"
 require 'optparse'
+require 'rake'
 
 ENV['LOGSTASH_PATH'] = File.expand_path('..', __dir__)
 ENV['LOGSTASH_SOURCE'] = '1'
@@ -41,19 +43,19 @@ def initialize(plugins_folder, plugin_name)
 
   def git_retrieve
     if File.directory?(plugin_name)
-      puts "#{plugin_name} already cloned locally, proceed with updating..."
+      puts "test plugins(git_retrieve)>> #{plugin_name} already cloned locally, proceed with updating... (at #{Time.new})"
       Dir.chdir(plugin_name) do
         system("git restore -- .")
         puts "Cleaning following files"
         system("git clean -n ")
         puts "Proceed with cleaning"
         system("git clean -Xf")
       end
-      puts "#{plugin_name} updated"
+      puts "test plugins(git_retrieve)>> #{plugin_name} updated"
       return
     end
 
-    puts "#{plugin_name} local clone doesn't exist, cloning..."
+    puts "test plugins(git_retrieve)>> #{plugin_name} local clone doesn't exist, cloning... (at #{Time.new})"
 
     plugin_repository = "[email protected]:logstash-plugins/#{plugin_name}.git"
     unless system("git clone #{plugin_repository}")
@@ -65,18 +67,42 @@ def git_retrieve
 
   # return true if successed
   def execute_rspec
-    if File.exists?("#{plugin_base_folder}/build.gradle")
-      system("#{plugin_base_folder}/gradlew vendor")
+    # setup JRUBY_OPTS env var to open access to expected modules
+    # the content is the same of the file https://github.com/logstash-plugins/.ci/blob/main/dockerjdk17.env
+    ENV['JRUBY_OPTS'] = "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED -J--add-opens=java.base/java.security=ALL-UNNAMED -J--add-opens=java.base/java.io=ALL-UNNAMED -J--add-opens=java.base/java.nio.channels=ALL-UNNAMED -J--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -J--add-opens=java.management/sun.management=ALL-UNNAMED -Xregexp.interruptible=true -Xcompile.invokedynamic=true -Xjit.threshold=0 -J-XX:+PrintCommandLineFlags -v -W1"
+    ENV['USER'] = "logstash"
+
+    puts "test plugins(execute_rspec)>> bundle install"
+    return false unless system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle install")
+
+    unit_test_folders = Dir.glob('spec/**/*')
+        .select {|f| File.directory? f}
+        .select{|f| not f.include?('integration')}
+        .join(" ")
+
+    puts "test plugins(execute_rspec)>> rake vendor (at #{Time.new})"
+    return false unless system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle exec rake vendor")
+
+    puts "test plugins(execute_rspec)>> exec rspec"
+    rspec_command = "#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle exec rspec #{unit_test_folders} --tag ~integration --tag ~secure_integration"
+    puts "\t\t executing: #{rspec_command}"
+    stdout, stderr, status = Open3.capture3(rspec_command)
+    if status != 0
+      puts "Error executing rspec"
+      puts "Stderr ----------------------"
+      puts stderr
+      puts "Stdout ----------------------"
+      puts stdout
+      puts "OEFStdout--------------------"
+      return false
     end
-    system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle install")
-    spec_result = system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle exec rspec")
-    #puts "DNABG>> spec_result #{spec_result}"
-    return spec_result ? true : false
+    return true
   end
 
   # Return nil in case of error or the file name of the generated gem file
   def build_gem
-    system("gem build #{plugin_name}.gemspec")
+    gem_command = "#{ENV['LOGSTASH_PATH']}/bin/ruby -S gem build #{plugin_name}.gemspec"
+    system(gem_command)
 
     gem_name = Dir.glob("#{plugin_name}*.gem").first
     unless gem_name
@@ -115,28 +141,33 @@ def install_gem(gem_file)
   :tier1 => {
     :input => ["logstash-input-azure_event_hubs", "logstash-input-beats", "logstash-input-elasticsearch", "logstash-input-file",
                "logstash-input-generator", "logstash-input-heartbeat", "logstash-input-http", "logstash-input-http_poller",
-               "logstash-input-redis", "logstash-input-s3", "logstash-input-stdin", "logstash-input-syslog", "logstash-input-udp",
-               "logstash-input-elastic_agent"],
+               "logstash-input-redis", "logstash-input-stdin", "logstash-input-syslog", "logstash-input-udp",
+             ],
     :codec => ["logstash-codec-avro", "logstash-codec-cef", "logstash-codec-es_bulk", "logstash-codec-json",
                "logstash-codec-json_lines", "logstash-codec-line", "logstash-codec-multiline", "logstash-codec-plain",
                "logstash-codec-rubydebug"],
     :filter => ["logstash-filter-cidr", "logstash-filter-clone", "logstash-filter-csv", "logstash-filter-date", "logstash-filter-dissect",
                 "logstash-filter-dns", "logstash-filter-drop", "logstash-filter-elasticsearch", "logstash-filter-fingerprint",
-                "logstash-filter-geoip", "logstash-filter-grok", "logstash-filter-http", "logstash-filter-json", "logstash-filter-kv",
+                "logstash-filter-geoip", "logstash-filter-grok", "logstash-filter-http", 
+                #"logstash-filter-json", # Commented because of issue https://github.com/logstash-plugins/logstash-filter-json/issues/55
+                "logstash-filter-kv",
                 "logstash-filter-memcached", "logstash-filter-mutate", "logstash-filter-prune", "logstash-filter-ruby",
                 "logstash-filter-sleep", "logstash-filter-split", "logstash-filter-syslog_pri", "logstash-filter-translate",
                 "logstash-filter-truncate", "logstash-filter-urldecode", "logstash-filter-useragent", "logstash-filter-uuid",
                 "logstash-filter-xml"],
     :output => ["logstash-output-elasticsearch", "logstash-output-email", "logstash-output-file", "logstash-output-http",
-                "logstash-output-redis", "logstash-output-s3", "logstash-output-stdout", "logstash-output-tcp", "logstash-output-udp"],
+                "logstash-output-redis", "logstash-output-stdout", "logstash-output-tcp", "logstash-output-udp"],
     :integration => ["logstash-integration-jdbc", "logstash-integration-kafka", "logstash-integration-rabbitmq",
-                     "logstash-integration-elastic_enterprise_search"]
+                     "logstash-integration-elastic_enterprise_search", "logstash-integration-aws"]
   },
   :tier2 => {
-    :input => ["logstash-input-couchdb_changes", "logstash-input-gelf", "logstash-input-graphite", "logstash-input-jms",
+    :input => [# "logstash-input-couchdb_changes", # Removed because of https://github.com/logstash-plugins/logstash-input-couchdb_changes/issues/51
+               "logstash-input-gelf", "logstash-input-graphite", "logstash-input-jms",
                   "logstash-input-snmp", "logstash-input-sqs", "logstash-input-twitter"],
     :codec => ["logstash-codec-collectd", "logstash-codec-dots", "logstash-codec-fluent", "logstash-codec-graphite",
-                  "logstash-codec-msgpack", "logstash-codec-netflow"],
+                  "logstash-codec-msgpack", 
+                  #{}"logstash-codec-netflow" # Commented because of issue https://github.com/logstash-plugins/logstash-codec-netflow/issues/203
+                ],
     :filter => ["logstash-filter-aggregate", "logstash-filter-de_dot", "logstash-filter-throttle"],
     :output => ["logstash-output-csv", "logstash-output-graphite"]
   },
@@ -168,15 +199,25 @@ def select_by_tiers_and_kinds(tiers, kinds)
 end
 
 def select_plugins_by_opts(options)
-  select_plugins = []
-  if options[:plugin]
-    select_plugins << options[:plugin]
-  else
-    selected_tiers = options.fetch(:tiers, [:tier1, :tier2, :unsupported])
-    selected_kinds = options.fetch(:kinds, [:input, :codec, :filter, :output, :integration])
-    select_plugins = select_plugins + select_by_tiers_and_kinds(selected_tiers, selected_kinds)
-  end
-  select_plugins
+  return options[:plugins] if options[:plugins]
+
+  selected_tiers = options.fetch(:tiers, [:tier1, :tier2, :unsupported])
+  selected_kinds = options.fetch(:kinds, [:input, :codec, :filter, :output, :integration])
+  selected_partition = options.fetch(:split, "1/1")
+
+  select_plugins = select_by_tiers_and_kinds(selected_tiers, selected_kinds)
+  return split_by_partition(select_plugins, selected_partition)
+end
+
+# Return the partion corresponding to the definition of the given list
+def split_by_partition(list, partition_definition)
+  slice = partition_definition.split('/')[0].to_i
+  num_slices = partition_definition.split('/')[1].to_i
+
+  slice_size = list.size / num_slices
+  slice_start = (slice - 1) * slice_size
+  slice_end = slice == num_slices ? -1 : slice * slice_size - 1
+  return list[slice_start..slice_end]
 end
 
 def snapshot_logstash_artifacts!
@@ -202,9 +243,10 @@ def setup_logstash_for_development
 end
 
 option_parser = OptionParser.new do |opts|
-  opts.on '-t', '--tiers tier1, tier2, unsupported', Array, 'Use to select which tier to test. If no provided mean "all"'
-  opts.on '-k', '--kinds input, codec, filter, output', Array, 'Use to select which kind of plugin to test. If no provided mean "all"'
-  opts.on '-pPLUGIN', '--plugin=PLUGIN', 'Use to select a specific plugin, conflict with either -t and -k'
+  opts.on '-t', '--tiers tier1,tier2,unsupported', Array, 'Use to select which tier to test. If no provided mean "all"'
+  opts.on '-k', '--kinds input,codec,filter,output', Array, 'Use to select which kind of plugin to test. If no provided mean "all"'
+  opts.on '-p', '--plugins plugin1,plugin2', Array, 'Use to select a specific set of plugins, conflict with either -t and -k'
+  opts.on '-sPARTITION', '--split=PARTITION', String, 'Use to partition the set of plugins to execute, for example -s 1/3 means "execute the first third of the selected plugin list"'
   opts.on '-h', '--halt', 'Halt immediately on first error'
 end
 options = {}
@@ -214,15 +256,17 @@ def setup_logstash_for_development
 
 plugins = select_plugins_by_opts(options)
 
+puts "test plugins(start)>> start at #{Time.new}"
+
 setup_logstash_for_development
 
 # save to local git for test isolation
 snapshot_logstash_artifacts!
 
 plugins.each do |plugin_name|
-   restore_logstash_from_snapshot
+  restore_logstash_from_snapshot
 
-   Dir.chdir(plugins_folder) do
+  status = Dir.chdir(plugins_folder) do
     plugin = Plugin.new(plugins_folder, plugin_name)
     plugin.git_retrieve
 
@@ -248,12 +292,12 @@ def setup_logstash_for_development
       end
       :success
     end
-
-    # any of the verification subtask terminated with error
-    if status == :error
-      # break looping on plugins if immediate halt
-      break if options[:halt]
-    end
+    status
+  end
+  # any of the verification subtask terminated with error
+  if status == :error
+    # break looping on plugins if immediate halt
+    break if options[:halt]
   end
 end
 

diff --git a/ci/test_plugins.sh b/ci/test_plugins.sh
@@ -1,7 +1,9 @@
-#!/bin/sh -ie
+#!/usr/bin/env bash
+set -euo pipefail
+
 export JRUBY_OPTS="-J-Xmx1g"
 export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8"
 
 ./gradlew assemble
 
-bin/ruby ci/test_plugins.rb $@
+vendor/jruby/bin/jruby ci/test_plugins.rb $@