Test DRA x86_64 build in container

.buildkite/dra_pipeline.yml

@@ -1,12 +1,26 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
-agents:
-  cpu: "2"
-  memory: "4Gi"
-  ephemeralStorage: "20Gi"
-
+## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml)
 steps:
-  - label: ":wave: Greetings"
-    command: "echo 'TODO: run DRA Logstash'"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.1"
+  - input: "Build parameters"
+    if: build.source != "schedule"
+    fields:
+      - text: "VERSION_QUALIFIER_OPT"
+        key: "VERSION_QUALIFIER_OPT"
+        default: ""
+        required: false
+        hint: "Optional version qualifier for built artifacts e.g.: alpha1,beta1"
+
+  - wait: ~
+    if: build.source != "schedule"
+
+  - label: ":pipeline: Generate steps"
+    command: |
+      set -eo pipefail
+
+      export WORKFLOW_TYPE="snapshot"
+      python3 -m pip install pyyaml
+      echo "--- Generating dynamic pipeline steps:"
+      python3 .buildkite/scripts/dra/generatesteps.py
+
+      python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload

.buildkite/hooks/pre-command

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DOCKER_REGISTRY="docker.elastic.co"
+DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+
+function docker_login {
+  DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+  DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+  docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+  unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
+}
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET
+}
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "logstash-dra-pipeline-ci" ]]; then
+
+  vq=$(buildkite-agent meta-data get VERSION_QUALIFIER_OPT --default "")
+  if [[ ! -z "$vq" ]]; then
+    echo "+++ Exporting VERSION_QUALIFIER_OPT to value [$vq]"
+    export VERSION_QUALIFIER_OPT=$vq
+  else
+    echo "+++ No value provided for VERSION_QUALIFIER_OPT leaving unset"
+  fi
+
+  if [[ "$BUILDKITE_STEP_KEY" == "logstash_publish_dra" ]]; then
+    echo "+++ Setting DRA params"
+    release_manager_login
+  fi
+fi

.buildkite/hooks/pre-exit

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eo pipefail
+
+source .buildkite/scripts/unset-secrets.sh
+
+if command -v docker &>/dev/null; then
+  DOCKER_REGISTRY="docker.elastic.co"
+  docker logout $DOCKER_REGISTRY
+fi

.buildkite/scripts/dra/dra_common.sh

@@ -0,0 +1,39 @@
+function info {
+    echo "--- INFO: $1"
+}
+
+function error {
+    echo "--- ERROR: $1"
+    exit 1
+}
+
+function save_docker_tarballs {
+    local arch="${1:?architecture required}"
+    local version="${2:?stack-version required}"
+    local images="logstash logstash-oss"
+    if [ "${arch}" != "aarch64" ]; then
+        # No logstash-ubi8 for AARCH64
+        images="logstash logstash-oss logstash-ubi8"
+    fi
+
+    for image in ${images}; do
+        tar_file="${image}-${version}-docker-image-${arch}.tar"
+        docker save -o "build/${tar_file}" \
+            "docker.elastic.co/logstash/${image}:${version}" || \
+            error "Unable to save tar file ${tar_file} for ${image} image."
+        # NOTE: if docker save exited with non-zero the error log already exited the script
+        gzip "build/${tar_file}"
+    done
+}
+
+# Since we are using the system jruby, we need to make sure our jvm process
+# uses at least 1g of memory, If we don't do this we can get OOM issues when
+# installing gems. See https://github.com/elastic/logstash/issues/5179
+export JRUBY_OPTS="-J-Xmx1g"
+
+# Extract the version number from the version.yml file
+# e.g.: 8.6.0
+# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
+STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
+
+info "Agent is running on architecture [$(uname -i)]"

.buildkite/scripts/dra/dra_docker.sh

@@ -0,0 +1,90 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/dra_common.sh
+
+# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            rake artifact:docker || error "artifact:docker build failed."
+            rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+        else
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    staging)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
+            RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+        else
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    *)
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Saving tar.gz for docker images"
+save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
+
+info "Generated Artifacts"
+for file in build/logstash-*; do shasum $file;done
+
+info "Uploading DRA artifacts in buildkite's artifact store ..."
+# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the dra_x86_64.sh
+images="logstash logstash-oss"
+if [ "$ARCH" != "aarch64" ]; then
+    # No logstash-ubi8 for AARCH64
+    images="logstash logstash-oss logstash-ubi8"
+fi
+for image in ${images}; do
+    buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
+done
+
+# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
+# overwritten when building aarch64 (or viceversa).
+if [ "$ARCH" != "aarch64" ]; then
+    for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do
+        buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
+    done
+fi
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"

.buildkite/scripts/dra/dra_upload.sh

@@ -0,0 +1,85 @@
+#!/bin/bash -iex
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/dra_common.sh
+PLAIN_STACK_VERSION=$STACK_VERSION
+
+# This is the branch selector that needs to be passed to the release-manager
+# It has to be the name of the branch which originates the artifacts.
+RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p'`
+if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then
+    RELEASE_BRANCH=$RELEASE_VER
+else
+    RELEASE_BRANCH=main
+fi
+
+if [ -n "$VERSION_QUALIFIER_OPT" ]; then
+  # Qualifier is passed from CI as optional field and specify the version postfix
+  # in case of alpha or beta releases:
+  # e.g: 8.0.0-alpha1
+  STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+  PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+fi
+
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        ;;
+    staging)
+        ;;
+    *)
+        error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
+
+if [ "$RELEASE_VER" != "7.17" ]; then
+  # Version 7.17.x doesn't generates ARM artifacts for Darwin
+  # TODO see if we need to do anything here
+fi
+
+# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected
+# by the release manager.
+# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle
+# for more details.
+# TODO filter it out when uploading artifacts instead
+rm build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
+
+info "Downloaded ARTIFACTS sha report"
+for file in build/logstash-*; do shasum $file;done
+
+mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv
+
+# set required permissions on artifacts and directory
+chmod -R a+r build/*
+chmod -R a+w build
+
+chmod -R a+r $PWD/*
+chmod -R a+w $PWD
+
+# ensure the latest image has been pulled
+docker pull docker.elastic.co/infra/release-manager:latest
+
+# collect the artifacts for use with the unified build
+docker run --rm \
+  --name release-manager \
+  -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \
+  -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \
+  -e VAULT_SECRET_ID="${VAULT_SECRET}" \
+  --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+  docker.elastic.co/infra/release-manager:latest \
+    cli collect \
+      --project logstash \
+      --branch ${RELEASE_BRANCH} \
+      --commit "$(git rev-parse HEAD)" \
+      --workflow "${WORKFLOW_TYPE}" \
+      --version "${PLAIN_STACK_VERSION}" \
+      --artifact-set main
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"

.buildkite/scripts/dra/dra_x86_64.sh

@@ -0,0 +1,58 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/dra_common.sh
+
+# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+        else
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    staging)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+        else
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    *)
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Generated Artifacts"
+for file in build/logstash-*; do shasum $file;done
+
+info "Creating dependencies report for ${STACK_VERSION}"
+mkdir -p build/distributions/dependencies-reports/
+bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
+
+info "Generated dependencies report"
+shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
+
+info "Uploading DRA artifacts in buildkite's artifact store ..."
+buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*"
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"