diff --git a/.buildkite/supported_plugins_test_pipeline.yml b/.buildkite/supported_plugins_test_pipeline.yml index 253798d19db..bf68ae5f4e9 100644 --- a/.buildkite/supported_plugins_test_pipeline.yml +++ b/.buildkite/supported_plugins_test_pipeline.yml @@ -10,8 +10,28 @@ steps: - group: "Tier1 plugins test group" key: "tier1_plugins" steps: - - label: "Test Tier1 inputs" - command: ./ci/test_plugins.sh -t tier1 -k input + - label: "Test input file plugin" + command: | + source .buildkite/scripts/common/vm-agent.sh + ./ci/test_plugins.sh -p logstash-input-file + # Workaround for https://github.com/elastic/ingest-dev/issues/2676 + agents: + provider: gcp + imageProject: elastic-images-prod + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-4" + diskSizeGb: 120 + - label: "Test rest of Tier1 inputs" + command: | + source .buildkite/scripts/common/vm-agent.sh + ./ci/test_plugins.sh -p logstash-input-azure_event_hubs,logstash-input-beats,logstash-input-elasticsearch,logstash-input-generator,logstash-input-heartbeat,logstash-input-http,logstash-input-http_poller,logstash-input-redis,logstash-input-stdin,logstash-input-syslog,logstash-input-udp + # Workaround for https://github.com/elastic/ingest-dev/issues/2676 + agents: + provider: gcp + imageProject: elastic-images-prod + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-4" + diskSizeGb: 120 - label: "Test Tier1 filters" command: ./ci/test_plugins.sh -t tier1 -k filter - label: "Test Tier1 codecs" @@ -19,7 +39,16 @@ steps: - label: "Test Tier1 outputs" command: ./ci/test_plugins.sh -t tier1 -k output - label: "Test Tier1 integrations" - command: ./ci/test_plugins.sh -t tier1 -k integration + command: | + source .buildkite/scripts/common/vm-agent.sh + ./ci/test_plugins.sh -t tier1 -k integration + # Workaround to avoid errors on chmod of files + agents: + provider: gcp + imageProject: elastic-images-prod + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-4" + diskSizeGb: 120 - group: "Tier2 plugins test group" key: "tier2_plugins" diff --git a/ci/test_plugins.rb b/ci/test_plugins.rb index 4b381f781e7..8ac57e9e4fa 100644 --- a/ci/test_plugins.rb +++ b/ci/test_plugins.rb @@ -1,13 +1,14 @@ # encoding: utf-8 # Test the -# - build (rspec), +# - build (rspec unit testing), # - packaging (gem build) # - and deploy (bin/logstash-plugin install) # of a plugins inside the current Logstash, using its JRuby # Usage example: # bin/ruby ci/test_supported_plugins.rb -p logstash-integration-jdbc # bin/ruby ci/test_supported_plugins.rb -t tier1 -k input,codec,integration +# bin/ruby ci/test_supported_plugins.rb -t tier1 -k input,codec,integration --split 1/3 # # The script uses OS's temp folder unless the environment variable LOGSTASH_PLUGINS_TMP is specified. # The path of such variable should be absolute. @@ -15,6 +16,7 @@ require "open3" require "set" require 'optparse' +require 'rake' ENV['LOGSTASH_PATH'] = File.expand_path('..', __dir__) ENV['LOGSTASH_SOURCE'] = '1' @@ -41,7 +43,7 @@ def initialize(plugins_folder, plugin_name) def git_retrieve if File.directory?(plugin_name) - puts "#{plugin_name} already cloned locally, proceed with updating..." + puts "test plugins(git_retrieve)>> #{plugin_name} already cloned locally, proceed with updating... (at #{Time.new})" Dir.chdir(plugin_name) do system("git restore -- .") puts "Cleaning following files" @@ -49,11 +51,11 @@ def git_retrieve puts "Proceed with cleaning" system("git clean -Xf") end - puts "#{plugin_name} updated" + puts "test plugins(git_retrieve)>> #{plugin_name} updated" return end - puts "#{plugin_name} local clone doesn't exist, cloning..." + puts "test plugins(git_retrieve)>> #{plugin_name} local clone doesn't exist, cloning... (at #{Time.new})" plugin_repository = "git@github.com:logstash-plugins/#{plugin_name}.git" unless system("git clone #{plugin_repository}") @@ -65,18 +67,42 @@ def git_retrieve # return true if successed def execute_rspec - if File.exists?("#{plugin_base_folder}/build.gradle") - system("#{plugin_base_folder}/gradlew vendor") + # setup JRUBY_OPTS env var to open access to expected modules + # the content is the same of the file https://github.com/logstash-plugins/.ci/blob/main/dockerjdk17.env + ENV['JRUBY_OPTS'] = "-J--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED -J--add-opens=java.base/java.security=ALL-UNNAMED -J--add-opens=java.base/java.io=ALL-UNNAMED -J--add-opens=java.base/java.nio.channels=ALL-UNNAMED -J--add-opens=java.base/sun.nio.ch=ALL-UNNAMED -J--add-opens=java.management/sun.management=ALL-UNNAMED -Xregexp.interruptible=true -Xcompile.invokedynamic=true -Xjit.threshold=0 -J-XX:+PrintCommandLineFlags -v -W1" + ENV['USER'] = "logstash" + + puts "test plugins(execute_rspec)>> bundle install" + return false unless system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle install") + + unit_test_folders = Dir.glob('spec/**/*') + .select {|f| File.directory? f} + .select{|f| not f.include?('integration')} + .join(" ") + + puts "test plugins(execute_rspec)>> rake vendor (at #{Time.new})" + return false unless system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle exec rake vendor") + + puts "test plugins(execute_rspec)>> exec rspec" + rspec_command = "#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle exec rspec #{unit_test_folders} --tag ~integration --tag ~secure_integration" + puts "\t\t executing: #{rspec_command}" + stdout, stderr, status = Open3.capture3(rspec_command) + if status != 0 + puts "Error executing rspec" + puts "Stderr ----------------------" + puts stderr + puts "Stdout ----------------------" + puts stdout + puts "OEFStdout--------------------" + return false end - system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle install") - spec_result = system("#{ENV['LOGSTASH_PATH']}/bin/ruby -S bundle exec rspec") - #puts "DNABG>> spec_result #{spec_result}" - return spec_result ? true : false + return true end # Return nil in case of error or the file name of the generated gem file def build_gem - system("gem build #{plugin_name}.gemspec") + gem_command = "#{ENV['LOGSTASH_PATH']}/bin/ruby -S gem build #{plugin_name}.gemspec" + system(gem_command) gem_name = Dir.glob("#{plugin_name}*.gem").first unless gem_name @@ -115,28 +141,33 @@ def install_gem(gem_file) :tier1 => { :input => ["logstash-input-azure_event_hubs", "logstash-input-beats", "logstash-input-elasticsearch", "logstash-input-file", "logstash-input-generator", "logstash-input-heartbeat", "logstash-input-http", "logstash-input-http_poller", - "logstash-input-redis", "logstash-input-s3", "logstash-input-stdin", "logstash-input-syslog", "logstash-input-udp", - "logstash-input-elastic_agent"], + "logstash-input-redis", "logstash-input-stdin", "logstash-input-syslog", "logstash-input-udp", + ], :codec => ["logstash-codec-avro", "logstash-codec-cef", "logstash-codec-es_bulk", "logstash-codec-json", "logstash-codec-json_lines", "logstash-codec-line", "logstash-codec-multiline", "logstash-codec-plain", "logstash-codec-rubydebug"], :filter => ["logstash-filter-cidr", "logstash-filter-clone", "logstash-filter-csv", "logstash-filter-date", "logstash-filter-dissect", "logstash-filter-dns", "logstash-filter-drop", "logstash-filter-elasticsearch", "logstash-filter-fingerprint", - "logstash-filter-geoip", "logstash-filter-grok", "logstash-filter-http", "logstash-filter-json", "logstash-filter-kv", + "logstash-filter-geoip", "logstash-filter-grok", "logstash-filter-http", + #"logstash-filter-json", # Commented because of issue https://github.com/logstash-plugins/logstash-filter-json/issues/55 + "logstash-filter-kv", "logstash-filter-memcached", "logstash-filter-mutate", "logstash-filter-prune", "logstash-filter-ruby", "logstash-filter-sleep", "logstash-filter-split", "logstash-filter-syslog_pri", "logstash-filter-translate", "logstash-filter-truncate", "logstash-filter-urldecode", "logstash-filter-useragent", "logstash-filter-uuid", "logstash-filter-xml"], :output => ["logstash-output-elasticsearch", "logstash-output-email", "logstash-output-file", "logstash-output-http", - "logstash-output-redis", "logstash-output-s3", "logstash-output-stdout", "logstash-output-tcp", "logstash-output-udp"], + "logstash-output-redis", "logstash-output-stdout", "logstash-output-tcp", "logstash-output-udp"], :integration => ["logstash-integration-jdbc", "logstash-integration-kafka", "logstash-integration-rabbitmq", - "logstash-integration-elastic_enterprise_search"] + "logstash-integration-elastic_enterprise_search", "logstash-integration-aws"] }, :tier2 => { - :input => ["logstash-input-couchdb_changes", "logstash-input-gelf", "logstash-input-graphite", "logstash-input-jms", + :input => [# "logstash-input-couchdb_changes", # Removed because of https://github.com/logstash-plugins/logstash-input-couchdb_changes/issues/51 + "logstash-input-gelf", "logstash-input-graphite", "logstash-input-jms", "logstash-input-snmp", "logstash-input-sqs", "logstash-input-twitter"], :codec => ["logstash-codec-collectd", "logstash-codec-dots", "logstash-codec-fluent", "logstash-codec-graphite", - "logstash-codec-msgpack", "logstash-codec-netflow"], + "logstash-codec-msgpack", + #{}"logstash-codec-netflow" # Commented because of issue https://github.com/logstash-plugins/logstash-codec-netflow/issues/203 + ], :filter => ["logstash-filter-aggregate", "logstash-filter-de_dot", "logstash-filter-throttle"], :output => ["logstash-output-csv", "logstash-output-graphite"] }, @@ -168,15 +199,25 @@ def select_by_tiers_and_kinds(tiers, kinds) end def select_plugins_by_opts(options) - select_plugins = [] - if options[:plugin] - select_plugins << options[:plugin] - else - selected_tiers = options.fetch(:tiers, [:tier1, :tier2, :unsupported]) - selected_kinds = options.fetch(:kinds, [:input, :codec, :filter, :output, :integration]) - select_plugins = select_plugins + select_by_tiers_and_kinds(selected_tiers, selected_kinds) - end - select_plugins + return options[:plugins] if options[:plugins] + + selected_tiers = options.fetch(:tiers, [:tier1, :tier2, :unsupported]) + selected_kinds = options.fetch(:kinds, [:input, :codec, :filter, :output, :integration]) + selected_partition = options.fetch(:split, "1/1") + + select_plugins = select_by_tiers_and_kinds(selected_tiers, selected_kinds) + return split_by_partition(select_plugins, selected_partition) +end + +# Return the partion corresponding to the definition of the given list +def split_by_partition(list, partition_definition) + slice = partition_definition.split('/')[0].to_i + num_slices = partition_definition.split('/')[1].to_i + + slice_size = list.size / num_slices + slice_start = (slice - 1) * slice_size + slice_end = slice == num_slices ? -1 : slice * slice_size - 1 + return list[slice_start..slice_end] end def snapshot_logstash_artifacts! @@ -202,9 +243,10 @@ def setup_logstash_for_development end option_parser = OptionParser.new do |opts| - opts.on '-t', '--tiers tier1, tier2, unsupported', Array, 'Use to select which tier to test. If no provided mean "all"' - opts.on '-k', '--kinds input, codec, filter, output', Array, 'Use to select which kind of plugin to test. If no provided mean "all"' - opts.on '-pPLUGIN', '--plugin=PLUGIN', 'Use to select a specific plugin, conflict with either -t and -k' + opts.on '-t', '--tiers tier1,tier2,unsupported', Array, 'Use to select which tier to test. If no provided mean "all"' + opts.on '-k', '--kinds input,codec,filter,output', Array, 'Use to select which kind of plugin to test. If no provided mean "all"' + opts.on '-p', '--plugins plugin1,plugin2', Array, 'Use to select a specific set of plugins, conflict with either -t and -k' + opts.on '-sPARTITION', '--split=PARTITION', String, 'Use to partition the set of plugins to execute, for example -s 1/3 means "execute the first third of the selected plugin list"' opts.on '-h', '--halt', 'Halt immediately on first error' end options = {} @@ -214,15 +256,17 @@ def setup_logstash_for_development plugins = select_plugins_by_opts(options) +puts "test plugins(start)>> start at #{Time.new}" + setup_logstash_for_development # save to local git for test isolation snapshot_logstash_artifacts! plugins.each do |plugin_name| - restore_logstash_from_snapshot + restore_logstash_from_snapshot - Dir.chdir(plugins_folder) do + status = Dir.chdir(plugins_folder) do plugin = Plugin.new(plugins_folder, plugin_name) plugin.git_retrieve @@ -248,12 +292,12 @@ def setup_logstash_for_development end :success end - - # any of the verification subtask terminated with error - if status == :error - # break looping on plugins if immediate halt - break if options[:halt] - end + status + end + # any of the verification subtask terminated with error + if status == :error + # break looping on plugins if immediate halt + break if options[:halt] end end diff --git a/ci/test_plugins.sh b/ci/test_plugins.sh index 0814afd2b1c..542bd5d669a 100755 --- a/ci/test_plugins.sh +++ b/ci/test_plugins.sh @@ -1,7 +1,9 @@ -#!/bin/sh -ie +#!/usr/bin/env bash +set -euo pipefail + export JRUBY_OPTS="-J-Xmx1g" export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" ./gradlew assemble -bin/ruby ci/test_plugins.rb $@ +vendor/jruby/bin/jruby ci/test_plugins.rb $@