Introduce docker for CI builds.

This commit includes:
* A base Dockerfile and script to push to a Docker repo
* A per-build Dockerfile (derived from the base)
* Updates to the test scripts to allow for more parallel builds
* Docker wrappers for the tests scripts
* Update for the integration test readme to manually run the tests
* Clean up the output of the Java tests
* Remove offline tag for tests (no longer needed that we don't use docker dependent services)

This commit does NOT include:
* Changes needed for the CI system to use Docker

Fixes #8223

.dockerignore

@@ -0,0 +1,4 @@
+**/.git
+build
+logs
+

Dockerfile

@@ -0,0 +1,38 @@
+FROM container-registry-test.elastic.co/logstash-test/logstash-base:latest
+
+RUN ln -s /tmp/vendor /opt/logstash/vendor
+
+ADD gradlew /opt/logstash/gradlew
+ADD gradle/wrapper /opt/logstash/gradle/wrapper
+RUN /opt/logstash/gradlew wrapper
+
+ADD versions.yml /opt/logstash/versions.yml
+ADD LICENSE /opt/logstash/LICENSE
+ADD CONTRIBUTORS /opt/logstash/CONTRIBUTORS
+ADD Gemfile.template /opt/logstash/Gemfile
+ADD Rakefile /opt/logstash/Rakefile
+ADD build.gradle /opt/logstash/build.gradle
+ADD rakelib /opt/logstash/rakelib
+ADD config /opt/logstash/config
+ADD spec /opt/logstash/spec
+ADD qa /opt/logstash/qa
+ADD lib /opt/logstash/lib
+ADD pkg /opt/logstash/pkg
+ADD tools /opt/logstash/tools
+ADD logstash-core /opt/logstash/logstash-core
+ADD logstash-core-plugin-api /opt/logstash/logstash-core-plugin-api
+ADD bin /opt/logstash/bin
+ADD modules /opt/logstash/modules
+ADD ci /opt/logstash/ci
+ADD CHANGELOG.md /opt/logstash/CHANGELOG.md
+ADD settings.gradle /opt/logstash/settings.gradle
+
+USER root
+RUN rm -rf build && \
+    mkdir -p build && \
+    chown -R logstash:logstash /opt/logstash
+USER logstash
+WORKDIR /opt/logstash
+
+LABEL retention="prune"
+

Dockerfile.base

@@ -0,0 +1,47 @@
+#logstash-base image, use ci/docker_update_base_image.sh to push updates
+FROM ubuntu:xenial
+
+RUN apt-get update && \
+    apt-get install -y zlib1g-dev build-essential vim rake git curl libssl-dev libreadline-dev libyaml-dev  \
+      libxml2-dev libxslt-dev openjdk-8-jdk-headless curl iputils-ping netcat && \
+    apt-get clean
+
+WORKDIR /root
+
+RUN adduser --disabled-password --gecos "" --home /home/logstash logstash && \
+    mkdir -p /usr/local/share/ruby-build && \
+    mkdir -p /opt/logstash && \
+    mkdir -p /mnt/host && \
+    chown logstash:logstash /opt/logstash
+
+USER logstash
+WORKDIR /home/logstash
+
+RUN git clone https://github.com/sstephenson/rbenv.git .rbenv && \
+    git clone https://github.com/sstephenson/ruby-build.git .rbenv/plugins/ruby-build && \
+    echo 'export PATH=/home/logstash/.rbenv/bin:$PATH' >> /home/logstash/.bashrc
+
+ENV PATH "/home/logstash/.rbenv/bin:$PATH"
+
+#Only used to help bootstrap the build (not to run Logstash itself)
+RUN echo 'eval "$(rbenv init -)"' >> .bashrc && \
+    rbenv install jruby-9.1.12.0 && \
+    rbenv install jruby-1.7.27 && \
+    rbenv global jruby-9.1.12.0 && \
+    bash -i -c 'gem install bundler' && \
+    rbenv local jruby-9.1.12.0 && \
+    mkdir -p /opt/logstash/data
+
+
+# Create a cache for the dependencies based on the current master, any dependencies not cached will be downloaded at runtime
+RUN git clone https://github.com/elastic/logstash.git /tmp/logstash && \
+    cd /tmp/logstash && \
+    rake test:install-core && \
+    ./gradlew compileJava compileTestJava && \
+    cd qa/integration && \
+    /home/logstash/.rbenv/shims/bundle install && \
+    mv /tmp/logstash/vendor /tmp/vendor && \
+    rm -rf /tmp/logstash
+
+# used by the purge policy
+LABEL retention="keep"

build.gradle

@@ -8,10 +8,46 @@ allprojects {
   project.targetCompatibility = JavaVersion.VERSION_1_8
 
   tasks.withType(JavaCompile).all {
-    options.compilerArgs.add("-Xlint:all")
+
+      def env = System.getenv()
+      boolean ci = env['CI']
+
+      //don't lint when running CI builds
+      if(!ci){
+          options.compilerArgs.add("-Xlint:all")
+      }
   }
 
   clean {
     delete "${projectDir}/out/"
   }
+
+  //https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time
+  tasks.withType(Test) {
+    testLogging {
+      // set options for log level LIFECYCLE
+      events "passed", "skipped", "failed", "standardOut"
+      showExceptions true
+      exceptionFormat "full"
+      showCauses true
+      showStackTraces true
+
+      // set options for log level DEBUG and INFO
+      debug {
+        events "started", "passed", "skipped", "failed", "standardOut", "standardError"
+        exceptionFormat "full"
+      }
+      info.events = debug.events
+      info.exceptionFormat = debug.exceptionFormat
+
+      afterSuite { desc, result ->
+        if (!desc.parent) { // will match the outermost suite
+          def output = "Results: ${result.resultType} (${result.testCount} tests, ${result.successfulTestCount} successes, ${result.failedTestCount} failures, ${result.skippedTestCount} skipped)"
+          def startItem = '|  ', endItem = '  |'
+          def repeatLength = startItem.length() + output.length() + endItem.length()
+          println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
+        }
+      }
+    }
+  }
 }

ci/acceptance_tests.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -e
+
+# Since we are using the system jruby, we need to make sure our jvm process
+# uses at least 1g of memory, If we don't do this we can get OOM issues when
+# installing gems. See https://github.com/elastic/logstash/issues/5179
+export JRUBY_OPTS="-J-Xmx1g"
+
+SELECTED_TEST_SUITE=$1
+
+# The acceptance test in our CI infrastructure doesn't clear the workspace between run
+# this mean the lock of the Gemfile can be sticky from a previous run, before generating any package
+# we will clear them out to make sure we use the latest version of theses files
+# If we don't do this we will run into gem Conflict error.
+[ -f Gemfile ] && rm Gemfile
+[ -f Gemfile.jruby-2.3.lock ] && rm Gemfile.jruby-2.3.lock
+
+if [[ $SELECTED_TEST_SUITE == $"redhat" ]]; then
+  echo "Generating the RPM, make sure you start with a clean environment before generating other packages."
+  rake artifact:rpm
+  echo "Acceptance: Installing dependencies"
+  cd qa
+  bundle install
+
+  echo "Acceptance: Running the tests"
+  bundle exec rake qa:vm:setup["redhat"]
+  bundle exec rake qa:vm:ssh_config
+  bundle exec rake qa:acceptance:redhat
+  bundle exec rake qa:vm:halt["redhat"]
+elif [[ $SELECTED_TEST_SUITE == $"debian" ]]; then
+  echo "Generating the DEB, make sure you start with a clean environment before generating other packages."
+  rake artifact:deb
+  echo "Acceptance: Installing dependencies"
+  cd qa
+  bundle install
+
+  echo "Acceptance: Running the tests"
+  bundle exec rake qa:vm:setup["debian"]
+  bundle exec rake qa:vm:ssh_config
+  bundle exec rake qa:acceptance:debian
+  bundle exec rake qa:vm:halt["debian"]
+elif [[ $SELECTED_TEST_SUITE == $"all" ]]; then
+  echo "Building Logstash artifacts"
+  rake artifact:all
+
+  echo "Acceptance: Installing dependencies"
+  cd qa
+  bundle install
+
+  echo "Acceptance: Running the tests"
+  bundle exec rake qa:vm:setup
+  bundle exec rake qa:vm:ssh_config
+  bundle exec rake qa:acceptance:all
+  bundle exec rake qa:vm:halt
+  cd ..
+fi

ci/docker_integration_tests.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -i
+#Note - ensure that the -e flag is NOT set, and explicitly check the $? status to allow for clean up
+
+if [ -z "$branch_specifier" ]; then
+    # manual
+    IMAGE_NAME="logstash-integration-tests"
+else
+    # Jenkins
+    IMAGE_NAME=$branch_specifier"-"$(date +%s%N)
+fi
+
+echo "Running Docker CI build for '$IMAGE_NAME' "
+
+docker build  -t $IMAGE_NAME .
+exit_code=$?; [[ $exit_code != 0 ]] && exit $exit_code
+docker run -t --rm $IMAGE_NAME ci/integration_tests.sh $@
+exit_code=$?
+[[ $IMAGE_NAME != "logstash-integration-tests" ]] && docker rmi $IMAGE_NAME
+echo "exiting with code: '$exit_code'"
+exit $exit_code #preserve the exit code from the test run
+

ci/docker_prune.sh

@@ -0,0 +1,6 @@
+#!/bin/bash -i
+
+echo "Removing containers older then 8 hours"
+docker container prune -f --filter "until=8h"
+echo "Removing all images, except with the label of retention=keep"
+docker image prune -a -f --filter "label!=retention=keep"

ci/docker_unit_tests.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -i
+#Note - ensure that the -e flag is NOT set, and explicitly check the $? status to allow for clean up
+
+if [ -z "$branch_specifier" ]; then
+    # manual
+    IMAGE_NAME="logstash-unit-tests"
+else
+    # Jenkins
+    IMAGE_NAME=$branch_specifier"-"$(date +%s%N)
+fi
+
+echo "Running Docker CI build for '$IMAGE_NAME' "
+
+docker build  -t $IMAGE_NAME .
+exit_code=$?; [[ $exit_code != 0 ]] && exit $exit_code
+docker run -t --rm $IMAGE_NAME ci/unit_tests.sh $@
+exit_code=$?
+[[ $IMAGE_NAME != "logstash-unit-tests" ]] && docker rmi $IMAGE_NAME
+echo "exiting with code: '$exit_code'"
+exit $exit_code #preserve the exit code from the test run

ci/docker_update_base_image.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -ie
+
+if docker rmi --force logstash-base ; then
+    echo "Removed existing logstash-base image, building logstash-base image from scratch."
+else
+    echo "Building logstash-base image from scratch." #Keep the global -e flag but allow the remove command to fail.
+fi
+
+docker build -f Dockerfile.base -t logstash-base .
+docker login --username=logstashci container-registry-test.elastic.co #will prompt for password
+docker tag logstash-base container-registry-test.elastic.co/logstash-test/logstash-base
+docker push container-registry-test.elastic.co/logstash-test/logstash-base

ci/integration_tests.sh

@@ -0,0 +1,53 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+
+# Since we are using the system jruby, we need to make sure our jvm process
+# uses at least 1g of memory, If we don't do this we can get OOM issues when
+# installing gems. See https://github.com/elastic/logstash/issues/5179
+export JRUBY_OPTS="-J-Xmx1g"
+
+export SPEC_OPTS="--order rand --format documentation"
+export CI=true
+
+rm -rf build && mkdir build
+
+echo "Building tar"
+rake artifact:tar
+cd build
+tar xf *.tar.gz
+
+cd ../qa/integration
+echo "Installing test dependencies"
+bundle install
+
+if [[ $1 = "setup" ]]; then
+ echo "Setup only, no tests will be run"
+ exit 0
+
+elif [[ $1 == "split" ]]; then
+    glob1=(specs/*spec.rb)
+    glob2=(specs/**/*spec.rb)
+    all_specs=("${glob1[@]}" "${glob2[@]}")
+
+    specs0=${all_specs[@]::$((${#all_specs[@]} / 2 ))}
+    specs1=${all_specs[@]:$((${#all_specs[@]} / 2 ))}
+
+    if [[ $2 == 0 ]]; then
+       echo "Running the first half of integration specs: $specs0"
+       bundle exec rspec $specs0
+    elif [[ $2 == 1 ]]; then
+       echo "Running the second half of integration specs: $specs1"
+       bundle exec rspec $specs1
+    else
+       echo "Error, must specify 0 or 1 after the split. For example ci/integration_tests.sh split 0"
+       exit 1
+    fi
+
+elif [[ !  -z  $@  ]]; then
+    echo "Running integration tests 'rspec $@'"
+    bundle exec rspec $@
+
+else
+    echo "Running all integration tests"
+    bundle exec rspec
+fi

ci/unit_tests.bat

@@ -0,0 +1,41 @@
+@echo off
+
+setlocal
+
+REM Since we are using the system jruby, we need to make sure our jvm process
+REM uses at least 1g of memory, If we don't do this we can get OOM issues when
+REM installing gems. See https://github.com/elastic/logstash/issues/5179
+
+SET JRUBY_OPTS="-J-Xmx1g"
+SET SELECTEDTESTSUITE=%1
+SET /p JRUBYVERSION=<.ruby-version
+
+IF NOT EXIST %JRUBYSRCDIR% (
+  echo "Variable JRUBYSRCDIR must be declared with a valid directory. Aborting.."
+  exit /B 1
+)
+
+SET JRUBYPATH=%JRUBYSRCDIR%\%JRUBYVERSION%
+
+IF NOT EXIST %JRUBYPATH% (
+  echo "Could not find JRuby in %JRUBYPATH%. Aborting.."
+  exit /B 1
+)
+
+SET RAKEPATH=%JRUBYPATH%\bin\rake
+
+IF "%SELECTEDTESTSUITE%"=="core-fail-fast" (
+  echo "Running core-fail-fast tests"
+  %RAKEPATH% test:install-core
+  %RAKEPATH% test:core-fail-fast
+) ELSE (
+  IF "%SELECTEDTESTSUITE%"=="all" (
+    echo "Running all plugins tests"
+    %RAKEPATH% test:install-all
+    %RAKEPATH% test:plugins
+  ) ELSE (
+    echo "Running core tests"
+    %RAKEPATH% test:install-core
+    %RAKEPATH% test:core
+  )
+)

ci/unit_tests.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+
+# Since we are using the system jruby, we need to make sure our jvm process
+# uses at least 1g of memory, If we don't do this we can get OOM issues when
+# installing gems. See https://github.com/elastic/logstash/issues/5179
+export JRUBY_OPTS="-J-Xmx1g"
+
+export SPEC_OPTS="--order rand --format documentation"
+export CI=true
+
+SELECTED_TEST_SUITE=$1
+
+if [[ $SELECTED_TEST_SUITE == $"core-fail-fast" ]]; then
+  echo "Running Java and Ruby unit tests, but will fail fast"
+  echo "Running test:install-core"
+  rake test:install-core
+  echo "Running test:core-fail-fast"
+  rake test:core-fail-fast
+elif [[ $SELECTED_TEST_SUITE == $"all" ]]; then
+  echo "Running all plugins tests"
+  echo "Running test:install-all"  # Install all plugins in this logstash instance, including development dependencies
+  rake test:install-all
+  echo "Running test:plugins"    # Run all plugins tests
+  rake test:plugins
+elif [[ $SELECTED_TEST_SUITE == $"java" ]]; then
+  echo "Running Java unit tests"
+  echo "Running test:core-java"
+  rake test:core-java
+elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then
+  echo "Running Ruby unit tests"
+  echo "Running test:install-core"
+  rake test:install-core
+  echo "Running test:core-ruby"
+  rake test:core-ruby
+else
+  echo "Running Java and Ruby unit tests"
+  echo "Running test:install-core"
+  rake test:install-core
+  echo "Running test:core"
+  rake test:core
+fi