From cebe4a7537a2b94368a7c183c9d777ea86c3f264 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 8 Jan 2024 09:40:58 +0200 Subject: [PATCH] Refactor qa/acceptance tests to get away from vagrant (#15696) This commit modernizes the qa/acceptance (packaging) test framework by moving away from Vagrant and having the tests operate locally. As we are migrating to Buildkite, the expectation is that those tests will run on dedicated vms thus removing the necessity of vagrant. Relates: https://github.com/elastic/ingest-dev/issues/1722 --- ci/acceptance_tests.sh | 102 ++++------- qa/Gemfile | 1 - qa/README.md | 172 ++++-------------- qa/Rakefile | 67 +------ qa/acceptance/helpers.rb | 55 ++++++ qa/acceptance/spec/config_helper.rb | 38 ---- .../spec/lib/artifact_composition_spec.rb | 33 ++-- .../spec/lib/artifact_operation_spec.rb | 9 +- qa/acceptance/spec/lib/cli_operation_spec.rb | 23 +-- .../cli/logstash-plugin/generate.rb | 2 +- .../cli/logstash-plugin/install.rb | 2 +- .../cli/logstash-plugin/integration_plugin.rb | 6 +- .../cli/logstash-plugin/list.rb | 2 +- .../cli/logstash-plugin/remove.rb | 2 +- .../cli/logstash-plugin/uninstall.rb | 2 +- .../cli/logstash-plugin/update.rb | 2 +- .../shared_examples/cli/logstash/version.rb | 2 +- .../spec/shared_examples/installed.rb | 6 +- .../shared_examples/installed_with_jdk.rb | 8 +- qa/acceptance/spec/shared_examples/running.rb | 2 +- qa/acceptance/spec/shared_examples/updated.rb | 13 +- qa/acceptance/spec/spec_helper.rb | 38 +--- qa/config/platforms.json | 16 -- qa/platform_config.rb | 99 ---------- qa/rspec/commands.rb | 154 ++++++++++------ qa/rspec/commands/base.rb | 91 +++++---- qa/rspec/commands/debian.rb | 38 ++-- qa/rspec/commands/opensuse.rb | 67 +++++++ qa/rspec/commands/redhat.rb | 39 ++-- qa/rspec/commands/suse.rb | 50 ++--- qa/rspec/commands/suse/sles-11.rb | 16 +- qa/rspec/commands/system_helpers.rb | 36 ++-- qa/rspec/helpers.rb | 54 ------ qa/rspec/matchers/be_installed.rb | 5 +- qa/rspec/matchers/be_running.rb | 3 +- qa/sys/debian/bootstrap.sh | 7 - qa/sys/debian/debian-8/bootstrap.sh | 6 - qa/sys/debian/ubuntu-1404/bootstrap.sh | 8 - qa/sys/debian/user_bootstrap.sh | 5 - qa/sys/redhat/bootstrap.sh | 4 - qa/sys/redhat/user_bootstrap.sh | 5 - qa/sys/suse/bootstrap.sh | 4 - qa/sys/suse/sles-11/bootstrap.sh | 8 - qa/sys/suse/sles-12/bootstrap.sh | 11 -- qa/sys/suse/user_bootstrap.sh | 4 - qa/vagrant/command.rb | 87 --------- qa/vagrant/helpers.rb | 71 -------- 47 files changed, 469 insertions(+), 1006 deletions(-) create mode 100644 qa/acceptance/helpers.rb delete mode 100644 qa/acceptance/spec/config_helper.rb delete mode 100644 qa/config/platforms.json delete mode 100644 qa/platform_config.rb create mode 100644 qa/rspec/commands/opensuse.rb delete mode 100644 qa/rspec/helpers.rb delete mode 100644 qa/sys/debian/bootstrap.sh delete mode 100644 qa/sys/debian/debian-8/bootstrap.sh delete mode 100644 qa/sys/debian/ubuntu-1404/bootstrap.sh delete mode 100644 qa/sys/debian/user_bootstrap.sh delete mode 100644 qa/sys/redhat/bootstrap.sh delete mode 100644 qa/sys/redhat/user_bootstrap.sh delete mode 100644 qa/sys/suse/bootstrap.sh delete mode 100644 qa/sys/suse/sles-11/bootstrap.sh delete mode 100644 qa/sys/suse/sles-12/bootstrap.sh delete mode 100644 qa/sys/suse/user_bootstrap.sh delete mode 100644 qa/vagrant/command.rb delete mode 100644 qa/vagrant/helpers.rb diff --git a/ci/acceptance_tests.sh b/ci/acceptance_tests.sh index f7b6dd98796..814eae5fe19 100755 --- a/ci/acceptance_tests.sh +++ b/ci/acceptance_tests.sh @@ -1,6 +1,19 @@ #!/usr/bin/env bash -set -e -set -x +set -eo pipefail + +function get_package_type { + # determines OS packaging system; at the moment either rpm or deb + source /etc/os-release + + if [[ $ID == "ubuntu" || $ID == "debian" || $ID_LIKE == "debian" ]]; then + PACKAGE_TYPE="deb" + elif [[ $ID_LIKE == *"rhel"* || $ID_LIKE == *"fedora"* || $ID_LIKE == *"suse"* ]]; then + PACKAGE_TYPE="rpm" + else + echo "^^^ +++ Unsupported Linux distribution [$ID]. Acceptance packaging tests only support deb or rpm based distributions. Exiting." + exit 1 + fi +} # Since we are using the system jruby, we need to make sure our jvm process # uses at least 1g of memory, If we don't do this we can get OOM issues when @@ -13,78 +26,33 @@ if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" fi - -SELECTED_TEST_SUITE=$1 - -# The acceptance test in our CI infrastructure doesn't clear the workspace between run -# this mean the lock of the Gemfile can be sticky from a previous run, before generating any package -# we will clear them out to make sure we use the latest version of theses files -# If we don't do this we will run into gem Conflict error. -[ -f Gemfile ] && rm Gemfile -[ -f Gemfile.lock ] && rm Gemfile.lock - -# When running these tests in a Jenkins matrix, in parallel, once one Vagrant job is done, the Jenkins ProcessTreeKiller will kill any other Vagrant processes with the same -# BUILD_ID unless you set this magic flag: https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller -export BUILD_ID=dontKillMe - LS_HOME="$PWD" QA_DIR="$PWD/qa" -# Always run the halt, even if the test times out or an exit is sent -cleanup() { - - cd $QA_DIR - bundle check || bundle install - bundle exec rake qa:vm:halt -} -trap cleanup EXIT - -# Cleanup any stale VMs from old jobs first -cleanup +cd $LS_HOME -if [[ $SELECTED_TEST_SUITE == $"redhat" ]]; then - echo "Generating the RPM, make sure you start with a clean environment before generating other packages." - cd $LS_HOME - ./gradlew clean bootstrap - rake artifact:rpm - echo "Acceptance: Installing dependencies" - cd $QA_DIR - bundle install +get_package_type - echo "Acceptance: Running the tests" - bundle exec rake qa:vm:setup["redhat"] - bundle exec rake qa:vm:ssh_config - bundle exec rake qa:acceptance:redhat - bundle exec rake qa:vm:halt["redhat"] -elif [[ $SELECTED_TEST_SUITE == $"debian" ]]; then - echo "Generating the DEB, make sure you start with a clean environment before generating other packages." - cd $LS_HOME +# in CI (Buildkite), packaging artifacts are pre-built from a previous step +if [[ $BUILDKITE == true ]]; then + export LS_ARTIFACTS_PATH="$HOME/build" + echo "--- Downloading artifacts from \"build/*${PACKAGE_TYPE}\" to $LS_ARTIFACTS_PATH" + set -x + # also creates build/ under $HOME + buildkite-agent artifact download "build/*${PACKAGE_TYPE}" $HOME + set +x + echo "--- Running gradle" ./gradlew clean bootstrap - rake artifact:deb - echo "Acceptance: Installing dependencies" - cd $QA_DIR - bundle install - - echo "Acceptance: Running the tests" - bundle exec rake qa:vm:setup["debian"] - bundle exec rake qa:vm:ssh_config - bundle exec rake qa:acceptance:debian - bundle exec rake qa:vm:halt["debian"] -elif [[ $SELECTED_TEST_SUITE == $"all" ]]; then - echo "Building Logstash artifacts" - cd $LS_HOME +else + echo "--- Detected a distribution that supports \033[33m[$PACKAGE_TYPE]\033[0m packages. Running gradle." ./gradlew clean bootstrap - rake artifact:all - - echo "Acceptance: Installing dependencies" - cd $QA_DIR - bundle install - - echo "Acceptance: Running the tests" - bundle exec rake qa:vm:setup - bundle exec rake qa:vm:ssh_config - bundle exec rake qa:acceptance:all - bundle exec rake qa:vm:halt + echo "--- Building Logstash artifacts" + rake artifact:$PACKAGE_TYPE fi +echo "--- Acceptance: Installing dependencies" +cd $QA_DIR +bundle install +echo "--- Acceptance: Running the tests" +rake qa:acceptance:all diff --git a/qa/Gemfile b/qa/Gemfile index 5b28063891a..90de65993b7 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -1,5 +1,4 @@ source "https://rubygems.org" -gem "runner-tool", :git => "https://github.com/purbon/runner-tool.git" gem "rspec", "~> 3.1.0" gem "rake" gem "stud" diff --git a/qa/README.md b/qa/README.md index c790da6d666..3063e1362e3 100644 --- a/qa/README.md +++ b/qa/README.md @@ -1,130 +1,44 @@ ## Acceptance test Framework -Welcome to the acceptance test framework for Logstash. In this small README we -describe its features and the steps necessary for setting up your environment. +The acceptance test framework for Logstash is intended to test the functionality of packages (`.deb`, `.rpm`) +on various supported platforms. -### Setup your environment +In this small README we describe its features and the steps necessary for executing it and adding new tests. -In summary this test framework is composed of: - -* A collection of rspec helpers and matchers that make creating tests - easy. -* This rspecs helpers execute commands over SSH to a set of machines. -* The tests are run, for now, as vagrant (virtualbox provided) machines. - -As of this, you need to have installed: - -* The latest version vagrant (=> 1.8.1) -* Virtualbox as VM provider (=> 5.0) - -Is important to notice that the first time you set everything up, or when a -new VM is added, there is the need to download the box (this will -take a while depending on your internet speed). - - -### Running Tests - -It is possible to run the full suite of the acceptance test with the codebase by -running the command `ci/acceptance_tests.sh`, this command will generate the artifacts, bootstrap -the VM and run the tests. +### Description -This test are based on a collection of Vagrant defined VM's where the -different test are going to be executed, so first setup necessary is to -have vagrant properly available, see https://www.vagrantup.com/ for -details on how to install it. - -_Inside the `qa` directory_ - -First of all execute the command `bundle` this will pull the necessary -dependencies in your environment, after this is done, this is the collection of task available for you: - -``` -skywalker% rake -T -rake qa:acceptance:all # Run all acceptance -rake qa:acceptance:debian # Run acceptance test in debian machines -rake qa:acceptance:redhat # Run acceptance test in redhat machines -rake qa:acceptance:single[machine] # Run one single machine acceptance test -rake qa:acceptance:suse # Run acceptance test in suse machines -rake qa:vm:halt[platform] # Halt all VM's involved in the acceptance test round -rake qa:vm:setup[platform] # Bootstrap all the VM's used for this tests -rake qa:vm:ssh_config # Generate a valid ssh-config -``` - -Important to be aware that using any of this commands: - -``` -rake qa:acceptance:all # Run all acceptance -rake qa:acceptance:debian # Run acceptance test in debian machines -rake qa:acceptance:redhat # Run acceptance test in redhat machines -rake qa:acceptance:suse # Run acceptance test in suse machines -``` - -before you *will have to bootstrap* all selected machines, you can do -that using the `rake qa:vm:setup[platform]` task. This is done like this -as bootstrap imply setting up the VM'S and this might take some time and -you might only want to this once. - -In the future we might add new rake tasks to do all at once, but for now you can use the script under -`ci/ci_acceptance.sh` to do all at once. +In summary this test framework is composed of: -For local testing purposes, is recommended to not run all together, pick your target and run with the single machine command, If you're willing to run on single one, you should use: +* A collection of rspec helpers and matchers that make creating tests easy. +* Rspecs helpers that execute commands. -``` -rake qa:acceptance:single[machine] # Run one single machine acceptance test -``` +The tests are expected to be executed on the target environment e.g. an Ubuntu 22.04 vm. -### How to run tests +### Running tests/Prerequisites -If you are *running this test for first time*, you will need to setup -your VM's first, you can do that using either `vagrant up` or `rake qa:vm:setup[platform]`. +To run the tests from a fresh Logstash checkout, you need to: -In this framework we're using ssh to connect to a collection of Vagrant -machines, so first and most important is to generate a valid ssh config -file, this could be done running `rake qa:vm:ssh_config`. When this task -is finished a file named `.vm_ssh_config` will be generated with all the -necessary information to connect with the different machines. +1. `./gradlew clean boostrap` +2a. Build the necessary package artifacts e.g. `rake artifact:deb` + **OR** +2b. Supply a directory where pregenerated package artifacts exit via the `LS_ARTIFACTS_PATH` environment variable (relative and absolute paths are supported). +3. `cd qa` +4. `bundle install` -Now is time to run your test and to do that we have different options: +Now you are ready to kick off the tests: -* rake qa:acceptance:all # Run all acceptance -* rake qa:acceptance:debian # Run acceptance test in debian machines -* rake qa:acceptance:redhat # Run acceptance test in redhat machines -* rake qa:acceptance:suse # Run acceptance test in suse machines -* rake qa:acceptance:single[machine] # Run one single machine acceptance test +5. `rake qa:acceptance:all`. -Generally speaking this are complex tests so they take a long time to -finish completely, if you look for faster feedback see at the end of this -README how to run fewer tests. +Steps 1, 2b, 3, 4, 5 are executed by the `ci/acceptance_tests.sh` script. ## Architecture of the Framework -If you wanna know more about how this framework works, here is your -section of information. - ### Directory structure -* ```acceptance/``` here it goes all the specs definitions. -* ```config``` inside you can find all config files, for now only the - platform definition. -* ```rspec``` here stay all framework parts necessary to get the test - running, you will find the commands, the rspec matchers and a -collection of useful helpers for your test. -* ```sys``` a collection of bash scripts used to bootstrap the machines. -* ```vagrant``` classes and modules used to help us running vagrant. - -### The platform configuration file - -Located inside the config directory there is the platforms.json which is used to define the different platforms we test with. -Important bits here are: - -* `latest` key defines the latest published version of LS release which is used to test the package upgrade scenario. -* inside the `platforms` key you will find the list of current available - OS we tests with, this include the box name, their type and if they -have to go under specific bootstrap scripts (see ```specific: true ``` -in the platform definition). - -This file is the one that you will use to know about different OS's -testes, add new ones, etc.. +* ```acceptance/```: all the specs definitions. +* ```rspec```: all framework parts necessary to get the test + running. Includes the commands, the rspec matchers and a +collection of useful helpers. ### I want to add a test, what should I do? @@ -143,45 +57,35 @@ including the different moving parts we encounter in the framework. ``` - config = ServiceTester.configuration - config.servers.each do |address| - ## - # ServiceTester::Artifact is the component used to interact with the - # destination machineri and the one that keep the necessary logic - # for it. - ## - - logstash = ServiceTester::Artifact.new(address, config.lookup[address]) + logstash = ServiceTester::Artifact.new() ## your test code goes here. - end + + # example: + it_behaves_like "installable_with_jdk", logstash + it_behaves_like "updated", logstash, from_release_branch="7.17" ``` -this is important because as you know we test with different machines, -so the build out artifact will be the component necessary to run the -actions with the destination machine. +Inside the `rspec` directory you will find a +collection of commands, organized per operating system, which +will let you operate and get your tests done. -but this is the main parts, to run your test you need the framework -located inside the ```rspec``` directory. Here you will find a -collection of commands, properly organized per operating system, that -will let you operate and get your tests done. But don't freak out, we -got all logic necessary to select the right one for your test. -You'll probably find enough supporting classes for different platforms, but if not, feel free to add it. +You'll probably find enough supporting classes for different platforms, but if not, feel free to add more. -FYI, this is how a command looks like: +An example of an install command on debian looks like: ``` - def installed?(hosts, package) + def installed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("dpkg -s #{package}") - stdout = cmd.stdout - end + cmd = sudo_exec!("dpkg -s #{package}") + stdout = cmd.stdout stdout.match(/^Package: #{package}$/) stdout.match(/^Status: install ok installed$/) + end end ``` + this is how we run operations and wrap them as ruby code. ### Running a test (detailed level) diff --git a/qa/Rakefile b/qa/Rakefile index 2aa2719549d..6af05e9a667 100644 --- a/qa/Rakefile +++ b/qa/Rakefile @@ -19,80 +19,15 @@ require "rspec" require "rspec/core/runner" require "rspec/core/rake_task" require "stud/try" -require_relative "vagrant/helpers" -require_relative "platform_config" - -platforms = PlatformConfig.new task :spec => 'spec:all' task :default => :spec namespace :qa do - namespace :vm do - def user_feedback_string_for(action, platform, machines, options = {}) - experimental_string = options["experimental"] ? "experimental" : "non experimental" - message = "#{action} all #{experimental_string} VM's defined in acceptance/Vagrantfile" - "#{message} for #{platform}: #{machines}" if !platform.nil? - end - - desc "Generate a valid ssh-config" - task :ssh_config do - require "json" - # Loop until the Vagrant box finishes SSH bootstrap - raw_ssh_config = Stud.try(50.times, LogStash::CommandExecutor::CommandError) do - LogStash::VagrantHelpers.fetch_config.stdout.split("\n"); - end - parsed_ssh_config = LogStash::VagrantHelpers.parse(raw_ssh_config) - File.write(".vm_ssh_config", parsed_ssh_config.to_json) - end - - desc "Bootstrap all the VM's used for this tests" - task :setup, :platform do |t, args| - config = PlatformConfig.new - experimental = (ENV['LS_QA_EXPERIMENTAL_OS'].to_s.downcase || "false") == "true" - machines = config.select_names_for(args[:platform], {"experimental" => experimental}) - - puts user_feedback_string_for("bootstrapping", args[:platform], machines, {"experimental" => experimental}) - - options = {:debug => ENV['LS_QA_DEBUG']} - puts "Destroying #{machines}" - LogStash::VagrantHelpers.destroy(machines, options) - puts "Bootstrapping #{machines}" - LogStash::VagrantHelpers.bootstrap(machines, options) - end - - desc "Halt all VM's involved in the acceptance test round" - task :halt, :platform do |t, args| - config = PlatformConfig.new - experimental = (ENV['LS_QA_EXPERIMENTAL_OS'].to_s.downcase || "false") == "true" - machines = config.select_names_for(args[:platform], {"experimental" => experimental}) - - puts user_feedback_string_for("halting", args[:platform], machines, {"experimental" => experimental}) - options = {:debug => ENV['LS_QA_DEBUG']} - - puts "Halting #{machines}" - LogStash::VagrantHelpers.halt(machines, options) - end - end - namespace :acceptance do - desc "Run all acceptance" + desc "Run all acceptance tests" task :all do exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/lib/**/*_spec.rb"]])) end - - platforms.types.each do |type| - desc "Run acceptance test in #{type} machines" - task type do - ENV['LS_TEST_PLATFORM'] = type - exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/lib/*_spec.rb"]])) - end - end - - desc "Run one single machine acceptance test" - task :single, :machine do |t, args| - ENV['LS_VAGRANT_HOST'] = args[:machine] - exit(RSpec::Core::Runner.run([Rake::FileList["acceptance/spec/lib/**/**/*_spec.rb"]])) - end end end diff --git a/qa/acceptance/helpers.rb b/qa/acceptance/helpers.rb new file mode 100644 index 00000000000..259cf4097ca --- /dev/null +++ b/qa/acceptance/helpers.rb @@ -0,0 +1,55 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require 'net/http' +require 'json' + +ARTIFACTS_API = "https://artifacts-api.elastic.co/v1/versions" + +def logstash_download_metadata(version, arch, artifact_type) + filename = "logstash-#{version}-#{arch}.#{artifact_type}" + { url: "https://artifacts.elastic.co/downloads/logstash/#{filename}", dest: File.join(ROOT, 'qa', filename) } +end + +def fetch_latest_logstash_release_version(branch) + uri = URI(ARTIFACTS_API) + + response = retryable_http_get(uri) + versions_data = JSON.parse(response) + + filtered_versions = versions_data["versions"].select { |v| v.start_with?(branch) && !v.include?('SNAPSHOT') } + + return filtered_versions.max_by { |v| Gem::Version.new(v) } +end + +def retryable_http_get(uri, max_retries=5, retry_wait=10) + count = 0 + + begin + response = Net::HTTP.get(uri) + rescue StandardError => e + count += 1 + if count < max_retries + puts "Retry attempt #{count}/#{max_retries}: #{e.message}" + sleep(retry_wait) + retry + else + puts "Exhausted all attempts trying to get from #{uri}." + raise e + end + end +end diff --git a/qa/acceptance/spec/config_helper.rb b/qa/acceptance/spec/config_helper.rb deleted file mode 100644 index 1b3a5b95ccb..00000000000 --- a/qa/acceptance/spec/config_helper.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "json" - -module SpecsHelper - def self.configure(vagrant_boxes) - setup_config = JSON.parse(File.read(File.join(File.dirname(__FILE__), "..", "..", ".vm_ssh_config"))) - boxes = vagrant_boxes.inject({}) do |acc, v| - acc[v.name] = v.type - acc - end - ServiceTester.configure do |config| - config.servers = [] - config.lookup = {} - setup_config.each do |host_info| - next unless boxes.keys.include?(host_info["host"]) - url = "#{host_info["hostname"]}:#{host_info["port"]}" - config.servers << url - config.lookup[url] = {"host" => host_info["host"], "type" => boxes[host_info["host"]] } - end - end - end -end diff --git a/qa/acceptance/spec/lib/artifact_composition_spec.rb b/qa/acceptance/spec/lib/artifact_composition_spec.rb index ef174460f3a..920e4b658dd 100644 --- a/qa/acceptance/spec/lib/artifact_composition_spec.rb +++ b/qa/acceptance/spec/lib/artifact_composition_spec.rb @@ -15,31 +15,28 @@ # specific language governing permissions and limitations # under the License. -require_relative '../spec_helper' +require_relative '../../../rspec/commands' describe "artifacts composition" do - config = ServiceTester.configuration - config.servers.each do |address| - logstash = ServiceTester::Artifact.new(address, config.lookup[address]) + logstash = ServiceTester::Artifact.new() - before(:each) do - logstash.install({:version => LOGSTASH_VERSION}) - end + before(:each) do + logstash.install({:version => LOGSTASH_VERSION}) + end - after(:each) do - logstash.uninstall - end + after(:each) do + logstash.uninstall + end - context 'prohibited gem dependencies' do - it 'does not vendor any version of kramdown' do - expect(logstash.gem_vendored?('kramdown')).to be false - end + context 'prohibited gem dependencies' do + it 'does not vendor any version of kramdown' do + expect(logstash.gem_vendored?('kramdown')).to be false end + end - context 'necessary gem dependencies (sanity check)' do - it 'vendors concurrent-ruby' do - expect(logstash.gem_vendored?('concurrent-ruby')).to be true - end + context 'necessary gem dependencies (sanity check)' do + it 'vendors concurrent-ruby' do + expect(logstash.gem_vendored?('concurrent-ruby')).to be true end end end diff --git a/qa/acceptance/spec/lib/artifact_operation_spec.rb b/qa/acceptance/spec/lib/artifact_operation_spec.rb index baf1c4b078a..710d3a592ef 100644 --- a/qa/acceptance/spec/lib/artifact_operation_spec.rb +++ b/qa/acceptance/spec/lib/artifact_operation_spec.rb @@ -21,10 +21,7 @@ # This tests verify that the generated artifacts could be used properly in a release, implements https://github.com/elastic/logstash/issues/5070 describe "artifacts operation" do - config = ServiceTester.configuration - config.servers.each do |address| - logstash = ServiceTester::Artifact.new(address, config.lookup[address]) - it_behaves_like "installable_with_jdk", logstash - it_behaves_like "updated", logstash - end + logstash = ServiceTester::Artifact.new() + it_behaves_like "installable_with_jdk", logstash + it_behaves_like "updated", logstash, from_release_branch="7.17" end diff --git a/qa/acceptance/spec/lib/cli_operation_spec.rb b/qa/acceptance/spec/lib/cli_operation_spec.rb index 4d73008b6cf..645bb857af2 100644 --- a/qa/acceptance/spec/lib/cli_operation_spec.rb +++ b/qa/acceptance/spec/lib/cli_operation_spec.rb @@ -28,18 +28,15 @@ # This is the collection of test for the CLI interface, this include the plugin manager behaviour, # it also include the checks for other CLI options. describe "CLI operation" do - config = ServiceTester.configuration - config.servers.each do |address| - logstash = ServiceTester::Artifact.new(address, config.lookup[address]) - # Force tests to use bundled JDK - logstash.run_command("unset LS_JAVA_HOME") - it_behaves_like "logstash version", logstash - it_behaves_like "logstash install", logstash - it_behaves_like "logstash list", logstash - it_behaves_like "logstash uninstall", logstash - it_behaves_like "logstash remove", logstash - it_behaves_like "logstash update", logstash - it_behaves_like "integration plugins compatible", logstash + logstash = ServiceTester::Artifact.new() + # Force tests to use bundled JDK + logstash.run_command("unset LS_JAVA_HOME") + it_behaves_like "logstash version", logstash + it_behaves_like "logstash install", logstash + it_behaves_like "logstash list", logstash + it_behaves_like "logstash uninstall", logstash + it_behaves_like "logstash remove", logstash + it_behaves_like "logstash update", logstash + it_behaves_like "integration plugins compatible", logstash # it_behaves_like "logstash generate", logstash - end end diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/generate.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/generate.rb index 0e6ae670850..f2b1b310e13 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/generate.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/generate.rb @@ -28,7 +28,7 @@ logstash.uninstall end - describe "on #{logstash.hostname}" do + describe "on [#{logstash.human_name}]" do GENERATE_TYPES = ["input", "filter", "codec", "output"] GENERATE_TYPES.each do |type| context "with type #{type}" do diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb index 0ff24a62549..a209dbb3413 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/install.rb @@ -28,7 +28,7 @@ logstash.uninstall end - describe "on #{logstash.hostname}" do + describe "on [#{logstash.human_name}]" do context "with a direct internet connection" do context "when the plugin exist" do context "from a local `.GEM` file" do diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/integration_plugin.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/integration_plugin.rb index 427a19b3e17..e2eb7d34ed3 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/integration_plugin.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/integration_plugin.rb @@ -20,7 +20,7 @@ require "fileutils" shared_examples "integration plugins compatible" do |logstash| - describe "logstash-plugin install on #{logstash.hostname}" do + describe "logstash-plugin install on [#{logstash.human_name}]" do let(:plugin) { "logstash-integration-rabbitmq" } before :each do logstash.install({:version => LOGSTASH_VERSION}) @@ -56,7 +56,7 @@ end end - describe "logstash-plugin uninstall on #{logstash.hostname}" do + describe "logstash-plugin uninstall on [#{logstash.human_name}]" do let(:plugin) { "logstash-integration-rabbitmq" } before :each do logstash.install({:version => LOGSTASH_VERSION}) @@ -79,7 +79,7 @@ end end - describe "logstash-plugin list on #{logstash.hostname}" do + describe "logstash-plugin list on [#{logstash.human_name}]" do let(:plugin) { "logstash-integration-rabbitmq" } before :each do logstash.install({:version => LOGSTASH_VERSION}) diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb index e910543bc38..45db82b7caf 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/list.rb @@ -20,7 +20,7 @@ require "fileutils" shared_examples "logstash list" do |logstash| - describe "logstash-plugin list on #{logstash.hostname}" do + describe "logstash-plugin list on [#{logstash.human_name}]" do before(:all) do logstash.install({:version => LOGSTASH_VERSION}) end diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb index 6c8a4685021..d05d16ccab2 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/remove.rb @@ -20,7 +20,7 @@ require "fileutils" shared_examples "logstash remove" do |logstash| - describe "logstash-plugin remove on #{logstash.hostname}" do + describe "logstash-plugin remove on [#{logstash.human_name}]" do before :each do logstash.install({:version => LOGSTASH_VERSION}) end diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb index a1f8aed3354..a1f1fe20e70 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/uninstall.rb @@ -20,7 +20,7 @@ require "fileutils" shared_examples "logstash uninstall" do |logstash| - describe "logstash-plugin uninstall on #{logstash.hostname}" do + describe "logstash-plugin uninstall on [#{logstash.human_name}]" do before :each do logstash.install({:version => LOGSTASH_VERSION}) end diff --git a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb index 1904fbf0011..45589217a92 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash-plugin/update.rb @@ -19,7 +19,7 @@ require "logstash/version" shared_examples "logstash update" do |logstash| - describe "logstash-plugin update on #{logstash.hostname}" do + describe "logstash-plugin update on [#{logstash.human_name}]" do before :each do logstash.install({:version => LOGSTASH_VERSION}) end diff --git a/qa/acceptance/spec/shared_examples/cli/logstash/version.rb b/qa/acceptance/spec/shared_examples/cli/logstash/version.rb index c05869b2b3a..22446ae1b3e 100644 --- a/qa/acceptance/spec/shared_examples/cli/logstash/version.rb +++ b/qa/acceptance/spec/shared_examples/cli/logstash/version.rb @@ -28,7 +28,7 @@ logstash.uninstall end - context "on #{logstash.hostname}" do + context "on [#{logstash.human_name}]" do it "returns the right logstash version" do result = logstash.run_command_in_path("bin/logstash --version") expect(result).to run_successfully_and_output(/#{LOGSTASH_VERSION}/) diff --git a/qa/acceptance/spec/shared_examples/installed.rb b/qa/acceptance/spec/shared_examples/installed.rb index 55a34c5ea48..09c7ee4bd07 100644 --- a/qa/acceptance/spec/shared_examples/installed.rb +++ b/qa/acceptance/spec/shared_examples/installed.rb @@ -25,17 +25,17 @@ logstash.install({:version => LOGSTASH_VERSION}) end - it "is installed on #{logstash.hostname}" do + it "is installed on [#{logstash.human_name}]" do expect(logstash).to be_installed end - it "is running on #{logstash.hostname}" do + it "is running on [#{logstash.human_name}]" do with_running_logstash_service(logstash) do expect(logstash).to be_running end end - it "is removable on #{logstash.hostname}" do + it "is removable on [#{logstash.human_name}]" do logstash.uninstall expect(logstash).to be_removed end diff --git a/qa/acceptance/spec/shared_examples/installed_with_jdk.rb b/qa/acceptance/spec/shared_examples/installed_with_jdk.rb index ed790eda8b1..280119fc342 100644 --- a/qa/acceptance/spec/shared_examples/installed_with_jdk.rb +++ b/qa/acceptance/spec/shared_examples/installed_with_jdk.rb @@ -18,7 +18,7 @@ require_relative '../spec_helper' require 'logstash/version' -# This test checks if a package is possible to be installed without errors. +# This test checks if a package can be installed without errors. RSpec.shared_examples "installable_with_jdk" do |logstash| before(:all) do #unset to force it using bundled JDK to run LS @@ -34,17 +34,17 @@ logstash.uninstall end - it "is installed on #{logstash.hostname}" do + it "is installed on [#{logstash.human_name}]" do expect(logstash).to be_installed end - it "is running on #{logstash.hostname}" do + it "is running on [#{logstash.human_name}]" do with_running_logstash_service(logstash) do expect(logstash).to be_running end end - it "is removable on #{logstash.hostname}" do + it "is removable on [#{logstash.human_name}]" do logstash.uninstall expect(logstash).to be_removed end diff --git a/qa/acceptance/spec/shared_examples/running.rb b/qa/acceptance/spec/shared_examples/running.rb index 9cd9eb22d0a..1fa2413f93c 100644 --- a/qa/acceptance/spec/shared_examples/running.rb +++ b/qa/acceptance/spec/shared_examples/running.rb @@ -24,7 +24,7 @@ logstash.install({:version => LOGSTASH_VERSION}) end - it "is running on #{logstash.hostname}" do + it "is running on [#{logstash.human_name}]" do with_running_logstash_service(logstash) do expect(logstash).to be_running end diff --git a/qa/acceptance/spec/shared_examples/updated.rb b/qa/acceptance/spec/shared_examples/updated.rb index d273f05dd64..211a69920c1 100644 --- a/qa/acceptance/spec/shared_examples/updated.rb +++ b/qa/acceptance/spec/shared_examples/updated.rb @@ -16,10 +16,11 @@ # under the License. require_relative '../spec_helper' +require_relative '../../helpers' require 'logstash/version' # This test checks if the current package could used to update from the latest version released. -RSpec.shared_examples "updated" do |logstash| +RSpec.shared_examples "updated" do |logstash, from_release_branch| before(:all) { #unset to force it using bundled JDK to run LS logstash.run_command("unset LS_JAVA_HOME") @@ -31,12 +32,14 @@ end before(:each) do - options = {:version => LOGSTASH_LATEST_VERSION, :snapshot => false, :base => "./", :skip_jdk_infix => true } - logstash.install(options) # make sure latest version is installed + latest_logstash_release_version = fetch_latest_logstash_release_version(from_release_branch) + url, dest = logstash_download_metadata(latest_logstash_release_version, logstash.client.architecture_extension, logstash.client.package_extension).values_at(:url, :dest) + logstash.download(url, dest) + options = {:version => latest_logstash_release_version, :snapshot => false, :base => "./", :skip_jdk_infix => false } + logstash.install(options) end - it "can be updated and run on #{logstash.hostname}" do - pending('Cannot install on OS') if logstash.hostname == 'oel-6' + it "can be updated and run on [#{logstash.human_name}]" do expect(logstash).to be_installed # Performing the update logstash.install({:version => LOGSTASH_VERSION}) diff --git a/qa/acceptance/spec/spec_helper.rb b/qa/acceptance/spec/spec_helper.rb index 4bd9fed00d5..5e41ad4b73a 100644 --- a/qa/acceptance/spec/spec_helper.rb +++ b/qa/acceptance/spec/spec_helper.rb @@ -15,46 +15,10 @@ # specific language governing permissions and limitations # under the License. -require 'runner-tool' -require_relative '../../rspec/helpers' -require_relative '../../rspec/matchers' -require_relative 'config_helper' -require_relative "../../platform_config" - -# This is a non obvious hack, -# EllipticalCurve are not completely implemented in JRuby 9k and the new version of SSH from the standard library -# use them. -# -# Details: https://github.com/jruby/jruby-openssl/issues/105 -Net::SSH::Transport::Algorithms::ALGORITHMS.values.each { |algs| algs.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } } -Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ } - ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..')) $LOAD_PATH.unshift File.join(ROOT, 'logstash-core/lib') -RunnerTool.configure - -RSpec.configure do |c| - c.include ServiceTester -end - -platform = ENV['LS_TEST_PLATFORM'] || 'all' -experimental = (ENV['LS_QA_EXPERIMENTAL_OS'].to_s.downcase || "false") == "true" - -config = PlatformConfig.new -LOGSTASH_LATEST_VERSION = config.latest - -default_vagrant_boxes = (platform == 'all' ? config.platforms : config.filter_type(platform, {"experimental" => experimental})) - -selected_boxes = if ENV.include?('LS_VAGRANT_HOST') then - config.platforms.select { |p| p.name == ENV['LS_VAGRANT_HOST'] } - else - default_vagrant_boxes - end - -SpecsHelper.configure(selected_boxes) - -puts "[Acceptance specs] running on #{ServiceTester.configuration.hosts}" if !selected_boxes.empty? +require_relative '../../rspec/matchers' def with_running_logstash_service(logstash) begin diff --git a/qa/config/platforms.json b/qa/config/platforms.json deleted file mode 100644 index 5b79e0a3277..00000000000 --- a/qa/config/platforms.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "latest": "5.0.0-alpha3", - "platforms" : { - "ubuntu-1604": { "box": "elastic/ubuntu-16.04-x86_64", "type": "debian" }, - "ubuntu-1804": { "box": "elastic/ubuntu-18.04-x86_64", "type": "debian" }, - "centos-7": { "box": "elastic/centos-7-x86_64", "type": "redhat" }, - "oel-7": { "box": "elastic/oraclelinux-7-x86_64", "type": "redhat" }, - "fedora-28": { "box": "elastic/fedora-28-x86_64", "type": "redhat", "experimental": true }, - "fedora-29": { "box": "elastic/fedora-29-x86_64", "type": "redhat", "experimental": true }, - "debian-8": { "box": "elastic/debian-8-x86_64", "type": "debian" }, - "debian-9": { "box": "elastic/debian-9-x86_64", "type": "debian" }, - "sles-11": { "box": "elastic/sles-11-x86_64", "type": "suse", "specific": true }, - "sles-12": { "box": "elastic/sles-12-x86_64", "type": "suse", "specific": true }, - "opensuse-13": { "box": "elastic/opensuse-13-x86_64", "type": "suse" } - } -} diff --git a/qa/platform_config.rb b/qa/platform_config.rb deleted file mode 100644 index 448f38a8a17..00000000000 --- a/qa/platform_config.rb +++ /dev/null @@ -1,99 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "json" -require "ostruct" - -# This is a wrapper to encapsulate the logic behind the different platforms we test with, -# this is done here in order to simplify the necessary configuration for bootstrap and interactions -# necessary later on in the tests phases. -# -class PlatformConfig - - # Abstract the idea of a platform, aka an OS - class Platform - - attr_reader :name, :box, :type, :bootstrap, :experimental - - def initialize(name, data) - @name = name - @box = data["box"] - @type = data["type"] - @experimental = data["experimental"] || false - configure_bootstrap_scripts(data) - end - - private - - def configure_bootstrap_scripts(data) - @bootstrap = OpenStruct.new(:privileged => "sys/#{type}/bootstrap.sh", - :non_privileged => "sys/#{type}/user_bootstrap.sh") - ## - # for now the only specific bootstrap scripts are ones need - # with privileged access level, whenever others are also - # required we can update this section as well with the same pattern. - ## - @bootstrap.privileged = "sys/#{type}/#{name}/bootstrap.sh" if data["specific"] - end - end - - DEFAULT_CONFIG_LOCATION = File.join(File.dirname(__FILE__), "config", "platforms.json").freeze - - attr_reader :platforms, :latest - - def initialize(config_path = DEFAULT_CONFIG_LOCATION) - @config_path = config_path - @platforms = [] - - data = JSON.parse(File.read(@config_path)) - data["platforms"].each do |k, v| - @platforms << Platform.new(k, v) - end - @platforms.sort! { |a, b| a.name <=> b.name } - @latest = data["latest"] - end - - def find!(platform_name) - result = @platforms.find { |platform| platform.name == platform_name }.first - if result.nil? - raise "Cannot find platform named: #{platform_name} in @config_path" - else - return result - end - end - - def each(&block) - @platforms.each(&block) - end - - def filter_type(type_name, options = {}) - experimental = options.fetch("experimental", false) - @platforms.select do |platform| - (type_name.nil? ? true : platform.type == type_name) && - platform.experimental == experimental - end - end - - def select_names_for(platform, options = {}) - filter_options = { "experimental" => options.fetch("experimental", false) } - filter_type(platform, filter_options).map { |p| p.name } - end - - def types - @platforms.collect(&:type).uniq.sort - end -end diff --git a/qa/rspec/commands.rb b/qa/rspec/commands.rb index 0348796339a..f4d9adbcf89 100644 --- a/qa/rspec/commands.rb +++ b/qa/rspec/commands.rb @@ -19,96 +19,152 @@ require_relative "./commands/ubuntu" require_relative "./commands/redhat" require_relative "./commands/suse" +require_relative "./commands/opensuse" require_relative "./commands/centos/centos-6" require_relative "./commands/oel/oel-6" require_relative "./commands/suse/sles-11" require "forwardable" +require "open3" + +OS_RELEASE_PATH = "/etc/os-release" + +class HostFacts + def initialize() + @os_release = {} + begin + os_release_hash = File.foreach(OS_RELEASE_PATH).each_with_object({}) do |line, hash| + next if line.strip.empty? + key, value = line.strip.split("=") + @os_release[key] = value.tr('"', "") + end + rescue Errno::ENOENT + puts "File not found: #{OS_RELEASE_PATH}" + rescue Errno::EACCES + puts "Permission denied to read file: #{OS_RELEASE_PATH}" + rescue StandardError => e + puts "Error parsing content of #{OS_RELEASE_PATH}: #{e.message}" + end + end + + def orig_name + # e.g. openSUSE Leap + @os_release["NAME"] + end + + def name + # e.g. opensuse leap + @os_release["NAME"].downcase + end + + def id + # e.g. ubuntu for Ubuntu 22.04, or debian for Debian 11, or centos for centos-7 + @os_release["ID"].downcase + end + + def id_like + # e.g. "rhel fedora" for centos-7 + @os_release["ID_LIKE"].downcase + end + + def version_codename + # e.g. jammy for Ubuntu 22.04, or bullseye for Debian 11, unset for RHEL + @os_release["VERSION_CODENAME"].downcase + end + + def version_id + # e.g. 22.04 for Ubuntu jammy, 11 for Debian Bullseye, 8.x for RHEL 8 distros + @os_release["VERSION_ID"].downcase + end + + def human_name + if self.version_id + "#{self.orig_name} #{self.version_id}" + else + orig_name + end + end +end module ServiceTester # An artifact is the component being tested, it's able to interact with # a destination machine by holding a client and is basically provides all # necessary abstractions to make the test simple. class Artifact - extend Forwardable def_delegators :@client, :installed?, :removed?, :running? - attr_reader :host, :client + attr_reader :client - def initialize(host, options = {}) - @host = host + def initialize(options = {}) @options = options - @client = CommandsFactory.fetch(options["type"], options["host"]) + @hostfacts = HostFacts.new() + @client = CommandsFactory.fetch(@hostfacts) @skip_jdk_infix = false end def hostname - @options["host"] + `hostname`.chomp end - def name - "logstash" + def human_name + @hostfacts.human_name end def hosts - [@host] - end - - def snapshot - client.snapshot(@options["host"]) + [@hostname] end - def restore - client.restore(@options["host"]) + def name + "logstash" end def start_service - client.start_service(name, host) + client.start_service(name) end def stop_service - client.stop_service(name, host) + client.stop_service(name) end def install(options = {}) base = options.fetch(:base, ServiceTester::Base::LOCATION) @skip_jdk_infix = options.fetch(:skip_jdk_infix, false) filename = filename(options) - package = client.package_for(filename, @skip_jdk_infix, base) - client.install(package, host) + package = client.package_for(filename, @skip_jdk_infix, base) + client.install(package) end def uninstall - client.uninstall(name, host) + client.uninstall(name) end def run_command_in_path(cmd) - client.run_command_in_path(cmd, host) + client.run_command_in_path(cmd) end def run_command(cmd) - client.run_command(cmd, host) + client.run_command(cmd) end def plugin_installed?(name, version = nil) - client.plugin_installed?(host, name, version) + client.plugin_installed?(name, version) end def gem_vendored?(gem_name) - client.gem_vendored?(host, gem_name) + client.gem_vendored?(gem_name) end def download(from, to) - client.download(from, to, host) + client.download(from, to) end def replace_in_gemfile(pattern, replace) - client.replace_in_gemfile(pattern, replace, host) + client.replace_in_gemfile(pattern, replace) end def delete_file(path) - client.delete_file(path, host) + client.delete_file(path) end def to_s @@ -118,39 +174,27 @@ def to_s private def filename(options = {}) - snapshot = options.fetch(:snapshot, true) + snapshot = options.fetch(:snapshot, true) "logstash-#{options[:version]}#{(snapshot ? "-SNAPSHOT" : "")}" end end - # Factory of commands used to select the right clients for a given type of OS and host name, - # this give you as much granularity as required. + # Factory of commands used to select the right clients for a given type of OS class CommandsFactory - - def self.fetch(type, host) - case type - when "debian" - if host.start_with?("ubuntu") - return UbuntuCommands.new - else - return DebianCommands.new - end - when "suse" - if host == "sles-11" - return Sles11Commands.new - else - return SuseCommands.new - end - when "redhat" - if host == "centos-6" - return Centos6Commands.new - elsif host == "oel-6" - return Oel6Commands.new - else - return RedhatCommands.new - end - else - return + def self.fetch(hostfacts) + case + when hostfacts.name.include?("ubuntu") + return UbuntuCommands.new + when hostfacts.name.include?("debian") + return DebianCommands.new + when hostfacts.name.include?("opensuse") + return OpenSuseCommands.new + when hostfacts.name.include?("red hat") + return RedhatCommands.new + when hostfacts.id_like.include?("rhel"), hostfacts.id_like.include?("fedora") + # covers Oracle Linux, CentOS, Rocky Linux, Amazon Linux + # TODO add specific commands (e.g. to use dnf instead of yum where applicable) + return RedhatCommands.new end end end diff --git a/qa/rspec/commands/base.rb b/qa/rspec/commands/base.rb index 1e3904d3e27..32ffb70cd99 100644 --- a/qa/rspec/commands/base.rb +++ b/qa/rspec/commands/base.rb @@ -16,58 +16,80 @@ # under the License. require 'tempfile' -require_relative "../../vagrant/helpers" +require 'open3' require_relative "system_helpers" +LS_BUILD_PATH = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'build')) + +class Command + def initialize() + @stdout, @stderr, @exit_status = nil + end + + def stdout + @stdout + end + + def stderr + @stderr + end + + def exit_status + @exit_status + end + + def execute(cmdline) + Open3.popen3(cmdline) do |stdin, stdout, stderr, wait_thr| + @stdout = stdout.read.chomp + @stderr = stderr.read.chomp + @exit_status = wait_thr.value.exitstatus + end + end +end + +def sudo_exec!(cmd) + command = Command.new() + command.execute("sudo #{cmd}") + return command +end + module ServiceTester class InstallException < Exception; end class Base - LOCATION = "/logstash-build".freeze + LOCATION = ENV.fetch('LS_ARTIFACTS_PATH', LS_BUILD_PATH.freeze) LOGSTASH_PATH = "/usr/share/logstash/".freeze - def snapshot(host) - LogStash::VagrantHelpers.save_snapshot(host) - end - - def restore(host) - LogStash::VagrantHelpers.restore_snapshot(host) - end - - def start_service(service, host = nil) - service_manager(service, "start", host) + def start_service(service) + service_manager(service, "start") end - def stop_service(service, host = nil) - service_manager(service, "stop", host) + def stop_service(service) + service_manager(service, "stop") end - def run_command(cmd, host) - hosts = (host.nil? ? servers : Array(host)) - + def run_command(cmd) response = nil - at(hosts, {in: :serial}) do |_host| - response = sudo_exec!("JARS_SKIP='true' #{cmd}") - end + response = sudo_exec!("JARS_SKIP='true' #{cmd}") response end - def replace_in_gemfile(pattern, replace, host) + def replace_in_gemfile(pattern, replace) gemfile = File.join(LOGSTASH_PATH, "Gemfile") cmd = "sed -i.sedbak 's/#{pattern}/#{replace}/' #{gemfile}" - run_command(cmd, host) + run_command(cmd) end - def run_command_in_path(cmd, host) - run_command("#{File.join(LOGSTASH_PATH, cmd)}", host) + def run_command_in_path(cmd) + run_command("#{File.join(LOGSTASH_PATH, cmd)}") end - def plugin_installed?(host, plugin_name, version = nil) + def plugin_installed?(plugin_name, version = nil) if version.nil? - cmd = run_command_in_path("bin/logstash-plugin list", host) + cmd = run_command_in_path("bin/logstash-plugin list") search_token = plugin_name else - cmd = run_command_in_path("bin/logstash-plugin list --verbose", host) + cmd = run_command_in_path("bin/logstash-plugin list --verbose") search_token = "#{plugin_name} (#{version})" end @@ -80,16 +102,15 @@ def plugin_installed?(host, plugin_name, version = nil) # # Returns `true` if _any version_ of the gem is vendored. # - # @param host [???] # @param gem_name [String] # @return [Boolean] # - the block should emit `true` iff the yielded gemspec meets the requirement, and `false` otherwise - def gem_vendored?(host, gem_name) - cmd = run_command("find /usr/share/logstash/vendor/bundle/jruby/*/specifications -name '#{gem_name}-*.gemspec'", host) + def gem_vendored?(gem_name) + cmd = run_command("find /usr/share/logstash/vendor/bundle/jruby/*/specifications -name '#{gem_name}-*.gemspec'") matches = cmd.stdout.lines matches.map do |path_to_gemspec| filename = path_to_gemspec.split('/').last - gemspec_contents = run_command("cat #{path_to_gemspec}", host).stdout + gemspec_contents = run_command("cat #{path_to_gemspec}").stdout Tempfile.create(filename) do |tempfile| tempfile.write(gemspec_contents) tempfile.flush @@ -98,12 +119,12 @@ def gem_vendored?(host, gem_name) end.select { |gemspec| gemspec.name == gem_name }.any? end - def download(from, to, host) - run_command("wget #{from} -O #{to}", host) + def download(from, to) + run_command("curl -fsSL --retry 5 --retry-delay 5 #{from} -o #{to}") end - def delete_file(path, host) - run_command("rm -rf #{path}", host) + def delete_file(path) + run_command("rm -rf #{path}") end def package_for(filename, skip_jdk_infix, base = ServiceTester::Base::LOCATION) diff --git a/qa/rspec/commands/debian.rb b/qa/rspec/commands/debian.rb index b4d57afaf65..a800bd9c64b 100644 --- a/qa/rspec/commands/debian.rb +++ b/qa/rspec/commands/debian.rb @@ -22,12 +22,10 @@ class DebianCommands < Base include ::ServiceTester::SystemD - def installed?(hosts, package) + def installed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("dpkg -s #{package}") - stdout = cmd.stdout - end + cmd = sudo_exec!("dpkg -s #{package}") + stdout = cmd.stdout stdout.match(/^Package: #{package}$/) stdout.match(/^Status: install ok installed$/) end @@ -44,32 +42,22 @@ def architecture_extension end end - def install(package, host = nil) - hosts = (host.nil? ? servers : Array(host)) - errors = [] - at(hosts, {in: :serial}) do |_| - cmd = sudo_exec!("dpkg -i --force-confnew #{package}") - if cmd.exit_status != 0 - errors << cmd.stderr.to_s - end + def install(package) + cmd = sudo_exec!("dpkg -i --force-confnew #{package}") + if cmd.exit_status != 0 + raise InstallException.new(cmd.stderr.to_s) end - raise InstallException.new(errors.join("\n")) unless errors.empty? end - def uninstall(package, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - sudo_exec!("dpkg -r #{package}") - sudo_exec!("dpkg --purge #{package}") - end + def uninstall(package) + sudo_exec!("dpkg -r #{package}") + sudo_exec!("dpkg --purge #{package}") end - def removed?(hosts, package) + def removed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("dpkg -s #{package}") - stdout = cmd.stderr - end + cmd = sudo_exec!("dpkg -s #{package}") + stdout = cmd.stderr ( stdout.match(/^Package `#{package}' is not installed and no info is available.$/) || stdout.match(/^dpkg-query: package '#{package}' is not installed and no information is available$/) diff --git a/qa/rspec/commands/opensuse.rb b/qa/rspec/commands/opensuse.rb new file mode 100644 index 00000000000..2f6b8d1ef71 --- /dev/null +++ b/qa/rspec/commands/opensuse.rb @@ -0,0 +1,67 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "base" + +module ServiceTester + class OpenSuseCommands < Base + + def installed?(package) + stdout = "" + cmd = sudo_exec!("zypper --no-refresh search #{package}") + stdout = cmd.stdout + stdout.match(/^i | logstash | An extensible logging pipeline | package$/) + end + + def package_extension() + "rpm" + end + + def architecture_extension() + "x86_64" + end + + def install(package) + cmd = sudo_exec!("zypper --no-gpg-checks --non-interactive install #{package}") + if cmd.exit_status != 0 + raise InstallException.new(cmd.stderr.to_s) + end + end + + def uninstall(package) + cmd = sudo_exec!("zypper --no-gpg-checks --non-interactive remove #{package}") + end + + def removed?(package) + stdout = "" + cmd = sudo_exec!("zypper --no-refresh info #{package}") + stdout = cmd.stdout + stdout.match(/package \'#{package}\' not found/) + end + + def running?(package) + stdout = "" + cmd = sudo_exec!("service #{package} status") + stdout = cmd.stdout + stdout.match(/Active: active \(running\)/) + end + + def service_manager(service, action) + sudo_exec!("service #{service} #{action}") + end + end +end diff --git a/qa/rspec/commands/redhat.rb b/qa/rspec/commands/redhat.rb index 07a7e2e13e4..cd2d1781791 100644 --- a/qa/rspec/commands/redhat.rb +++ b/qa/rspec/commands/redhat.rb @@ -22,12 +22,10 @@ class RedhatCommands < Base include ::ServiceTester::SystemD - def installed?(hosts, package) + def installed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = exec!("yum list installed #{package}") - stdout = cmd.stdout - end + cmd = sudo_exec!("yum list installed #{package}") + stdout = cmd.stdout stdout.match(/^Installed Packages$/) stdout.match(/^logstash.noarch/) || stdout.match(/^logstash.#{architecture_extension}/) end @@ -44,33 +42,22 @@ def architecture_extension end end - def install(package, host = nil) - hosts = (host.nil? ? servers : Array(host)) - errors = [] - exit_status = 0 - at(hosts, {in: :serial}) do |_host| - cmd = sudo_exec!("yum install -y #{package}") - exit_status += cmd.exit_status - errors << cmd.stderr unless cmd.stderr.empty? - end - if exit_status > 0 - raise InstallException.new("Error installing #{package}, #{errors.join('\n')}") + def install(package) + cmd = sudo_exec!("yum install -y #{package}") + if cmd.exit_status != 0 + raise InstallException.new(cmd.stderr.to_s) end + end - def uninstall(package, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - sudo_exec!("yum remove -y #{package}") - end + def uninstall(package) + sudo_exec!("yum remove -y #{package}") end - def removed?(hosts, package) + def removed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("yum list installed #{package}") - stdout = cmd.stderr - end + cmd = sudo_exec!("yum list installed #{package}") + stdout = cmd.stderr stdout.match(/^Error: No matching Packages to list$/) end end diff --git a/qa/rspec/commands/suse.rb b/qa/rspec/commands/suse.rb index 34e5ed0b5d6..c2db2e03801 100644 --- a/qa/rspec/commands/suse.rb +++ b/qa/rspec/commands/suse.rb @@ -20,12 +20,10 @@ module ServiceTester class SuseCommands < Base - def installed?(hosts, package) + def installed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = exec!("zypper search #{package}") - stdout = cmd.stdout - end + cmd = sudo_exec!("zypper search #{package}") + stdout = cmd.stdout stdout.match(/^i | logstash | An extensible logging pipeline | package$/) end @@ -37,47 +35,33 @@ def architecture_extension() "x86_64" end - def install(package, host = nil) - hosts = (host.nil? ? servers : Array(host)) - errors = [] - at(hosts, {in: :serial}) do |_host| - cmd = sudo_exec!("zypper --no-gpg-checks --non-interactive install #{package}") - errors << cmd.stderr unless cmd.stderr.empty? + def install(package) + cmd = sudo_exec!("zypper --no-gpg-checks --non-interactive install #{package}") + if cmd.exit_status != 0 + raise InstallException.new(cmd.stderr.to_s) end - raise InstallException.new(errors.join("\n")) unless errors.empty? end - def uninstall(package, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - cmd = sudo_exec!("zypper --no-gpg-checks --non-interactive remove #{package}") - end + def uninstall(package) + cmd = sudo_exec!("zypper --no-gpg-checks --non-interactive remove #{package}") end - def removed?(hosts, package) + def removed?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = exec!("zypper search #{package}") - stdout = cmd.stdout - end + cmd = sudo_exec!("zypper search #{package}") + stdout = cmd.stdout stdout.match(/No packages found/) end - def running?(hosts, package) + def running?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("service #{package} status") - stdout = cmd.stdout - end + cmd = sudo_exec!("service #{package} status") + stdout = cmd.stdout stdout.match(/Active: active \(running\)/) end - def service_manager(service, action, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - sudo_exec!("service #{service} #{action}") - end + def service_manager(service, action) + sudo_exec!("service #{service} #{action}") end - end end diff --git a/qa/rspec/commands/suse/sles-11.rb b/qa/rspec/commands/suse/sles-11.rb index 98774745958..af2c41845b6 100644 --- a/qa/rspec/commands/suse/sles-11.rb +++ b/qa/rspec/commands/suse/sles-11.rb @@ -21,21 +21,15 @@ module ServiceTester class Sles11Commands < SuseCommands - def running?(hosts, package) + def running?(package) stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("/etc/init.d/#{package} status") - stdout = cmd.stdout - end + cmd = sudo_exec!("/etc/init.d/#{package} status") + stdout = cmd.stdout stdout.match(/#{package} is running$/) end - def service_manager(service, action, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - sudo_exec!("/etc/init.d/#{service} #{action}") - end + def service_manager(service, action) + sudo_exec!("/etc/init.d/#{service} #{action}") end - end end diff --git a/qa/rspec/commands/system_helpers.rb b/qa/rspec/commands/system_helpers.rb index 85b8068da0a..3c3da8a2c4f 100644 --- a/qa/rspec/commands/system_helpers.rb +++ b/qa/rspec/commands/system_helpers.rb @@ -19,12 +19,10 @@ module ServiceTester module SystemD - def running?(hosts, package, jdk_path = '/usr/share/logstash/jdk/bin/java') + def running?(package, jdk_path = '/usr/share/logstash/jdk/bin/java') stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("service #{package} status") - stdout = cmd.stdout - end + cmd = sudo_exec!("service #{package} status") + stdout = cmd.stdout stdout.force_encoding(Encoding::UTF_8) ( stdout.match(/Active: active \(running\)/) && @@ -33,38 +31,28 @@ def running?(hosts, package, jdk_path = '/usr/share/logstash/jdk/bin/java') ) end - def service_manager(service, action, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - sudo_exec!("service #{service} #{action}") - end + def service_manager(service, action) + sudo_exec!("service #{service} #{action}") end end module InitD - def running?(hosts, package, jdk_path = '/usr/share/logstash/jdk/bin/java') + def running?(package, jdk_path = '/usr/share/logstash/jdk/bin/java') stdout = "" - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("initctl status #{package}") - stdout = cmd.stdout - end + cmd = sudo_exec!("initctl status #{package}") + stdout = cmd.stdout running = stdout.match(/#{package} start\/running/) if running pid = stdout.match(/#{package} start\/running, process (\d*)/).captures[0] - at(hosts, {in: :serial}) do |host| - cmd = sudo_exec!("ps ax | grep #{pid}") - stdout = cmd.stdout - end + cmd = sudo_exec!("ps ax | grep #{pid}") + stdout = cmd.stdout running = (running && stdout.match(/#{jdk_path}/)) end running end - def service_manager(service, action, host = nil) - hosts = (host.nil? ? servers : Array(host)) - at(hosts, {in: :serial}) do |_| - sudo_exec!("initctl #{action} #{service}") - end + def service_manager(service, action) + sudo_exec!("initctl #{action} #{service}") end end end diff --git a/qa/rspec/helpers.rb b/qa/rspec/helpers.rb deleted file mode 100644 index 0aa52c6eec0..00000000000 --- a/qa/rspec/helpers.rb +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require_relative "commands" - -module ServiceTester - class Configuration - attr_accessor :servers, :lookup - - def initialize - @servers = [] - @lookup = {} - end - - def hosts - lookup.values.map { |val| val["host"] } - end - end - - class << self - attr_accessor :configuration - end - - def self.configure - self.configuration ||= Configuration.new - yield(configuration) if block_given? - end - - def servers - ServiceTester.configuration.servers - end - - def select_client - CommandsFactory.fetch(current_example.metadata[:platform]) - end - - def current_example - RSpec.respond_to?(:current_example) ? RSpec.current_example : self.example - end -end diff --git a/qa/rspec/matchers/be_installed.rb b/qa/rspec/matchers/be_installed.rb index ea2c9990cbb..2829c55aac5 100644 --- a/qa/rspec/matchers/be_installed.rb +++ b/qa/rspec/matchers/be_installed.rb @@ -16,16 +16,15 @@ # under the License. require 'rspec/expectations' -require_relative '../helpers' RSpec::Matchers.define :be_installed do match do |subject| - subject.installed?(subject.hosts, subject.name) + subject.installed?(subject.name) end end RSpec::Matchers.define :be_removed do match do |subject| - subject.removed?(subject.hosts, subject.name) + subject.removed?(subject.name) end end diff --git a/qa/rspec/matchers/be_running.rb b/qa/rspec/matchers/be_running.rb index c0277769313..6347419b1be 100644 --- a/qa/rspec/matchers/be_running.rb +++ b/qa/rspec/matchers/be_running.rb @@ -16,10 +16,9 @@ # under the License. require 'rspec/expectations' -require_relative '../helpers' RSpec::Matchers.define :be_running do match do |subject| - subject.running?(subject.hosts, subject.name) + subject.running?(subject.name) end end diff --git a/qa/sys/debian/bootstrap.sh b/qa/sys/debian/bootstrap.sh deleted file mode 100644 index 29c2c840346..00000000000 --- a/qa/sys/debian/bootstrap.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -add-apt-repository ppa:openjdk-r/ppa -apt-get update -apt-get install -y openjdk-8-jdk -update-alternatives --config java -update-alternatives --config javac diff --git a/qa/sys/debian/debian-8/bootstrap.sh b/qa/sys/debian/debian-8/bootstrap.sh deleted file mode 100644 index da56514aae8..00000000000 --- a/qa/sys/debian/debian-8/bootstrap.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list -puts "installing jdk8" -apt-get update -apt-get install -y ca-certificates-java openjdk-8-jdk-headless diff --git a/qa/sys/debian/ubuntu-1404/bootstrap.sh b/qa/sys/debian/ubuntu-1404/bootstrap.sh deleted file mode 100644 index 728b0c3d13f..00000000000 --- a/qa/sys/debian/ubuntu-1404/bootstrap.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -add-apt-repository ppa:openjdk-r/ppa -apt-get update -apt-get install -y openjdk-8-jdk -update-alternatives --config java -update-alternatives --config javac -update-ca-certificates -f diff --git a/qa/sys/debian/user_bootstrap.sh b/qa/sys/debian/user_bootstrap.sh deleted file mode 100644 index a99d4ff056f..00000000000 --- a/qa/sys/debian/user_bootstrap.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -VERSION=`cat /vagrant/config/platforms.json | grep latest | cut -d":" -f2 | sed 's/["\|,| ]//g'` -LOGSTASH_FILENAME="logstash-${VERSION}.deb" -wget -q https://download.elastic.co/logstash/logstash/packages/debian/$LOGSTASH_FILENAME diff --git a/qa/sys/redhat/bootstrap.sh b/qa/sys/redhat/bootstrap.sh deleted file mode 100644 index 5976f47722f..00000000000 --- a/qa/sys/redhat/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -yum update -yum install -y java-1.8.0-openjdk-devel.x86_64 diff --git a/qa/sys/redhat/user_bootstrap.sh b/qa/sys/redhat/user_bootstrap.sh deleted file mode 100644 index 4713b3f5c1b..00000000000 --- a/qa/sys/redhat/user_bootstrap.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -VERSION=`cat /vagrant/config/platforms.json | grep latest | cut -d":" -f2 | sed 's/["\|,| ]//g'` -LOGSTASH_FILENAME="logstash-${VERSION}.rpm" -wget -q https://download.elastic.co/logstash/logstash/packages/centos/$LOGSTASH_FILENAME diff --git a/qa/sys/suse/bootstrap.sh b/qa/sys/suse/bootstrap.sh deleted file mode 100644 index 4dba83eb9ea..00000000000 --- a/qa/sys/suse/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -zypper --non-interactive list-updates -zypper --non-interactive --no-gpg-checks --quiet install --no-recommends java-1_8_0-openjdk-devel diff --git a/qa/sys/suse/sles-11/bootstrap.sh b/qa/sys/suse/sles-11/bootstrap.sh deleted file mode 100644 index 654be5d7ec0..00000000000 --- a/qa/sys/suse/sles-11/bootstrap.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -zypper rr systemsmanagement_puppet -zypper addrepo -t yast2 http://demeter.uni-regensburg.de/SLES12-x64/DVD1/ dvd1 || true -zypper addrepo -t yast2 http://demeter.uni-regensburg.de/SLES12-x64/DVD2/ dvd2 || true -ln -s /usr/sbin/update-alternatives /usr/sbin/alternatives -curl -L 'https://edelivery.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.rpm' -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0' -H 'Cookie: oraclelicense=accept-securebackup-cookie;' -H 'Connection: keep-alive' --compressed -o oracle_jdk_1.8.rpm -zypper -q -n --non-interactive install oracle_jdk_1.8.rpm diff --git a/qa/sys/suse/sles-12/bootstrap.sh b/qa/sys/suse/sles-12/bootstrap.sh deleted file mode 100644 index 56b4d0fd7d6..00000000000 --- a/qa/sys/suse/sles-12/bootstrap.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -zypper rr systemsmanagement_puppet -zypper addrepo -t yast2 http://demeter.uni-regensburg.de/SLES12-x64/DVD1/ dvd1 || true -zypper addrepo -t yast2 http://demeter.uni-regensburg.de/SLES12-x64/DVD2/ dvd2 || true -zypper addrepo http://download.opensuse.org/repositories/Java:Factory/SLE_12/Java:Factory.repo || true -zypper --no-gpg-checks --non-interactive refresh -zypper --non-interactive list-updates -ln -s /usr/sbin/update-alternatives /usr/sbin/alternatives -curl -L 'https://edelivery.oracle.com/otn-pub/java/jdk/8u77-b03/jdk-8u77-linux-x64.rpm' -H 'Accept-Encoding: gzip, deflate, sdch' -H 'Accept-Language: en-US,en;q=0.8' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0' -H 'Cookie: oraclelicense=accept-securebackup-cookie;' -H 'Connection: keep-alive' --compressed -o oracle_jdk_1.8.rpm -zypper -q -n --non-interactive install oracle_jdk_1.8.rpm diff --git a/qa/sys/suse/user_bootstrap.sh b/qa/sys/suse/user_bootstrap.sh deleted file mode 100644 index be22af8b0c1..00000000000 --- a/qa/sys/suse/user_bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -VERSION=`cat /vagrant/config/platforms.json | grep latest | cut -d":" -f2 | sed 's/["\|,| ]//g'` -LOGSTASH_FILENAME="logstash-${VERSION}.rpm" -wget -q https://download.elastic.co/logstash/logstash/packages/centos/$LOGSTASH_FILENAME diff --git a/qa/vagrant/command.rb b/qa/vagrant/command.rb deleted file mode 100644 index a0d4655901c..00000000000 --- a/qa/vagrant/command.rb +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "open3" -require "bundler" - -module LogStash - class CommandExecutor - class CommandError < StandardError; end - - class CommandResponse - attr_reader :stdin, :stdout, :stderr, :exitstatus - - def initialize(stdin, stdout, stderr, exitstatus) - @stdin = stdin - @stdout = stdout - @stderr = stderr - @exitstatus = exitstatus - end - - def success? - exitstatus == 0 - end - end - - def self.run(cmd, debug = false) - # This block is require to be able to launch a ruby subprocess - # that use bundler. - Bundler.with_clean_env do - stdin, stdout, stderr, wait_thr = Open3.popen3(cmd) - stdout_acc, stderr_acc = "", "" - stdout_reporter = reporter(stdout, wait_thr) do |c| - stdout_acc << c - print c if debug - end - reporter(stderr, wait_thr) do |c| - stderr_acc << c; - print c if debug - end - stdout_reporter.join - CommandResponse.new(stdin, stdout_acc, stderr_acc, wait_thr.value.exitstatus) - end - end - - # This method will raise an exception if the `CMD` - # was not run successfully and will display the content of STDERR - def self.run!(cmd, debug = false) - response = run(cmd, debug) - - unless response.success? - raise CommandError, "CMD: #{cmd} STDERR: #{response.stderr}, stdout: #{response.stdout}" - end - response - end - - private - - def self.reporter(io, wait_thr, &block) - Thread.new(io, wait_thr) do |_io, _wait_thr| - while (_wait_thr.status == "run" || _wait_thr.status == "sleep") - begin - c = _io.read(1) - block.call(c) if c - rescue IO::WaitReadable - IO.select([_io]) - retry - end - end - end - end - - end -end diff --git a/qa/vagrant/helpers.rb b/qa/vagrant/helpers.rb deleted file mode 100644 index d56739b1d4c..00000000000 --- a/qa/vagrant/helpers.rb +++ /dev/null @@ -1,71 +0,0 @@ -# Licensed to Elasticsearch B.V. under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch B.V. licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -require "open3" -require "bundler" -require_relative "command" - -module LogStash - class VagrantHelpers - - def self.halt(machines = [], options = {}) - debug = options.fetch(:debug, false) - CommandExecutor.run!("vagrant halt #{machines.join(' ')}", debug) - end - - def self.destroy(machines = [], options = {}) - debug = options.fetch(:debug, false) - CommandExecutor.run!("vagrant destroy --force #{machines.join(' ')}", debug) - end - - def self.bootstrap(machines = [], options = {}) - debug = options.fetch(:debug, false) - CommandExecutor.run!("vagrant up #{machines.join(' ')}", debug) - end - - def self.save_snapshot(machine = "") - CommandExecutor.run!("vagrant snapshot save #{machine} #{machine}-snapshot") - end - - def self.restore_snapshot(machine = "") - CommandExecutor.run!("vagrant snapshot restore #{machine} #{machine}-snapshot") - end - - def self.fetch_config - machines = CommandExecutor.run!("vagrant status --machine-readable").stdout.split("\n").select { |l| l.include?("state,running") }.map { |r| r.split(',')[1]} - CommandExecutor.run!("vagrant ssh-config #{machines.join(' ')}") - end - - def self.parse(lines) - hosts, host = [], {} - lines.each do |line| - if line.match(/Host\s(.*)$/) - host = { :host => line.gsub("Host", "").strip } - elsif line.match(/HostName\s(.*)$/) - host[:hostname] = line.gsub("HostName", "").strip - elsif line.match(/Port\s(.*)$/) - host[:port] = line.gsub("Port", "").strip - elsif line.empty? - hosts << host - host = {} - end - end - hosts << host - hosts - end - end -end