refactor travis tests for the new release cadence

.ci/docker-setup.sh

@@ -1,61 +1,88 @@
 #!/bin/bash
 
-# This is intended to be run the plugin's root directory. `.ci/docker-setup.sh`
+# This is intended to be run the plugin's root directory. `ci/unit/docker-test.sh`
 # Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable.
 set -e
 
-VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json"
-
-
 pull_docker_snapshot() {
   project="${1?project name required}"
-  local docker_image="docker.elastic.co/${project}/${project}:${ELASTIC_STACK_VERSION}"
+  local docker_image="docker.elastic.co/${project}/${project}${DISTRIBUTION_SUFFIX}:${ELASTIC_STACK_VERSION}"
   echo "Pulling $docker_image"
-  docker pull "$docker_image"
+  if docker pull "$docker_image" ; then
+    echo "docker pull successful"
+  else
+    case "$ELASTIC_STACK_VERSION_ARG" in
+      "8.previous"|"8.current"|"8.next")
+        exit 1
+        ;;
+      *)
+        exit 2
+        ;;
+    esac
+  fi
 }
 
-if [ "$ELASTIC_STACK_VERSION" ]; then
-    echo "Fetching versions from $VERSION_URL"
-    VERSIONS=$(curl --silent $VERSION_URL)
-    if [[ "$SNAPSHOT" = "true" ]]; then
-      ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
-      echo $ELASTIC_STACK_RETRIEVED_VERSION
-    else
-      ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"')
-    fi
-    if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then
-      # remove starting and trailing double quotes
-      ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}"
-      ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}"
-      echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}"
-      export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION
-    fi
+VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
+
+if [ -z "${ELASTIC_STACK_VERSION}" ]; then
+    echo "Please set the ELASTIC_STACK_VERSION environment variable"
+    echo "For example: export ELASTIC_STACK_VERSION=7.x"
+    exit 1
+fi
 
-    echo "Testing against version: $ELASTIC_STACK_VERSION"
+# save the original arg if needed
+ELASTIC_STACK_VERSION_ARG="$ELASTIC_STACK_VERSION"
 
-    if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then
-        pull_docker_snapshot "logstash"
-        if [ "$INTEGRATION" == "true" ]; then
-          pull_docker_snapshot "elasticsearch"
-        fi
-    fi
+echo "Fetching versions from $VERSION_URL"
+VERSIONS=$(curl -s $VERSION_URL)
 
-    if [ -f Gemfile.lock ]; then
-        rm Gemfile.lock
-    fi
+if [[ "$SNAPSHOT" = "true" ]]; then
+  ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
+  echo $ELASTIC_STACK_RETRIEVED_VERSION
+else
+  ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"')
+fi
 
-    cd .ci
+if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then
+  # remove starting and trailing double quotes
+  ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}"
+  ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}"
+  echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}"
+  export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION
+elif [[ "$ELASTIC_STACK_VERSION" == "8.next" ]]; then
+  # we know "8.next" only exists between FF and GA of a minor
+  # exit 1 so the build is skipped
+  exit 1
+fi
+
+case "${DISTRIBUTION}" in
+  default) DISTRIBUTION_SUFFIX="" ;; # empty string when explicit "default" is given
+        *) DISTRIBUTION_SUFFIX="${DISTRIBUTION/*/-}${DISTRIBUTION}" ;;
+esac
+export DISTRIBUTION_SUFFIX
+
+echo "Testing against version: $ELASTIC_STACK_VERSION (distribution: ${DISTRIBUTION:-"default"})"
 
+if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then
+    pull_docker_snapshot "logstash"
     if [ "$INTEGRATION" == "true" ]; then
-        docker-compose down
-        docker-compose build --quiet
-    else
-        docker-compose down
-        docker-compose build logstash --quiet
+      pull_docker_snapshot "elasticsearch"
     fi
-else
-    echo "Please set the ELASTIC_STACK_VERSION environment variable"
-    echo "For example: export ELASTIC_STACK_VERSION=6.2.4"
-    exit 1
 fi
 
+if [ -f Gemfile.lock ]; then
+    rm Gemfile.lock
+fi
+
+CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}")
+
+cd .ci
+
+export BUILDKIT_PROGRESS=plain
+if [ "$INTEGRATION" == "true" ]; then
+    docker-compose down
+    docker-compose build --quiet
+else
+    docker-compose down
+    docker-compose build logstash --quiet
+fi

.ci/logstash-run.sh

@@ -22,7 +22,7 @@ wait_for_es() {
     [[ $count -eq 0 ]] && exit 1
     sleep 1
   done
-  echo $(curl $CURL_OPTS -vi $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
+  echo $(curl $CURL_OPTS -vi $ES_URL | jq -r .version.number)
 }
 
 if [[ "$INTEGRATION" != "true" ]]; then

.travis.yml

@@ -1,13 +1,17 @@
 import:
 - logstash-plugins/.ci:travis/[email protected]
 
-env:
-  - INTEGRATION=false ELASTIC_STACK_VERSION=7.x
-  - INTEGRATION=false ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
-  - INTEGRATION=true ELASTIC_STACK_VERSION=7.x
-  - INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info
-  - INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
-  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
-  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info
-  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_KEY_INVALID=true LOG_LEVEL=info
-  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 LOG_LEVEL=info
+jobs:
+  include:
+    - stage: "Integration Tests"
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.previous
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.future
+#    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main
+    - stage: "Secure Integration Tests"
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current SNAPSHOT=true
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_KEY_INVALID=true
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3