[ci] Backport Buildkite DRA pipeline+scripts to 8.10

This is a backport of the DRA pipeline and related scripts from:

- elastic#15366
- elastic#15365
- elastic#15356
- elastic#15352
- elastic#15344
- elastic#15343
- elastic#15337
- elastic#15312

.buildkite/dra/build_docker.sh

@@ -0,0 +1,90 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/common.sh
+
+# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            rake artifact:docker || error "artifact:docker build failed."
+            rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+        else
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    staging)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
+            RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+        else
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    *)
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Saving tar.gz for docker images"
+save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
+
+info "Generated Artifacts"
+for file in build/logstash-*; do shasum $file;done
+
+info "Uploading DRA artifacts in buildkite's artifact store ..."
+# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh
+images="logstash logstash-oss"
+if [ "$ARCH" != "aarch64" ]; then
+    # No logstash-ubi8 for AARCH64
+    images="logstash logstash-oss logstash-ubi8"
+fi
+for image in ${images}; do
+    buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
+done
+
+# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
+# overwritten when building aarch64 (or viceversa).
+if [ "$ARCH" != "aarch64" ]; then
+    for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do
+        buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
+    done
+fi
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"

.buildkite/dra/build_packages.sh

@@ -0,0 +1,58 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/common.sh
+
+# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+        else
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    staging)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+        else
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    *)
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Generated Artifacts"
+for file in build/logstash-*; do shasum $file;done
+
+info "Creating dependencies report for ${STACK_VERSION}"
+mkdir -p build/distributions/dependencies-reports/
+bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
+
+info "Generated dependencies report"
+shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
+
+info "Uploading DRA artifacts in buildkite's artifact store ..."
+buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*"
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"

.buildkite/dra/common.sh

@@ -0,0 +1,46 @@
+function info {
+    echo "--- INFO: $1"
+}
+
+function error {
+    echo "--- ERROR: $1"
+    exit 1
+}
+
+function save_docker_tarballs {
+    local arch="${1:?architecture required}"
+    local version="${2:?stack-version required}"
+    local images="logstash logstash-oss"
+    if [ "${arch}" != "aarch64" ]; then
+        # No logstash-ubi8 for AARCH64
+        images="logstash logstash-oss logstash-ubi8"
+    fi
+
+    for image in ${images}; do
+        tar_file="${image}-${version}-docker-image-${arch}.tar"
+        docker save -o "build/${tar_file}" \
+            "docker.elastic.co/logstash/${image}:${version}" || \
+            error "Unable to save tar file ${tar_file} for ${image} image."
+        # NOTE: if docker save exited with non-zero the error log already exited the script
+        gzip "build/${tar_file}"
+    done
+}
+
+# Since we are using the system jruby, we need to make sure our jvm process
+# uses at least 1g of memory, If we don't do this we can get OOM issues when
+# installing gems. See https://github.com/elastic/logstash/issues/5179
+export JRUBY_OPTS="-J-Xmx1g"
+
+# Extract the version number from the version.yml file
+# e.g.: 8.6.0
+# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
+STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
+
+info "Agent is running on architecture [$(uname -i)]"
+
+export VERSION_QUALIFIER_OPT=${VERSION_QUALIFIER_OPT:-""}
+export DRA_DRY_RUN=${DRA_DRY_RUN:-""}
+
+if [[ ! -z $DRA_DRY_RUN && $BUILDKITE_STEP_KEY == "logstash_publish_dra" ]]; then
+    info "Release manager will run in dry-run mode [$DRA_DRY_RUN]"
+fi

.buildkite/dra/docker-env-setup.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DOCKER_REGISTRY="docker.elastic.co"
+DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+
+function docker_login {
+  DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+  DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+  docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+  unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
+}
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID VAULT_SECRET_ID
+}

.buildkite/dra/docker-env-teardown.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Unset all variables ending with _SECRET or _TOKEN
+for var in $(printenv | sed 's;=.*;;' | sort); do
+  if [[ $var != "VAULT_ADDR" && ("$var" == *_SECRET || "$var" == *_TOKEN || "$var" == *VAULT* ) ]]; then
+      unset "$var"
+  fi
+done
+
+if command -v docker &>/dev/null; then
+  DOCKER_REGISTRY="docker.elastic.co"
+  docker logout $DOCKER_REGISTRY
+fi

.buildkite/dra/generatesteps.py

@@ -0,0 +1,131 @@
+import os
+import sys
+
+import yaml
+
+def to_bk_key_friendly_string(key):
+    """
+    Convert and return key to an acceptable format for Buildkite's key: field
+    Only alphanumerics, dashes and underscores are allowed.
+    """
+
+    mapping_table = str.maketrans({'.': '_'})
+
+    return key.translate(mapping_table)
+
+def package_x86_step(branch, workflow_type):
+    step = f'''
+- label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts"
+  key: "logstash_build_packages_dra"
+  agents:
+    provider: gcp
+    imageProject: elastic-images-prod
+    image: family/platform-ingest-logstash-ubuntu-2204
+    machineType: "n2-standard-16"
+    diskSizeGb: 200
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/build_packages.sh
+'''
+
+    return step
+
+def package_x86_docker_step(branch, workflow_type):
+    step = f'''
+- label: ":package: Build x86_64 Docker / {branch}-{workflow_type.upper()} DRA artifacts"
+  key: "logstash_build_x86_64_docker_dra"
+  agents:
+    provider: gcp
+    imageProject: elastic-images-prod
+    image: family/platform-ingest-logstash-ubuntu-2204
+    machineType: "n2-standard-16"
+    diskSizeGb: 200
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    export ARCH="x86_64"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/build_docker.sh
+'''
+
+    return step
+
+def package_aarch64_docker_step(branch, workflow_type):
+    step = f'''
+- label: ":package: Build aarch64 Docker / {branch}-{workflow_type.upper()} DRA artifacts"
+  key: "logstash_build_aarch64_docker_dra"
+  agents:
+    provider: aws
+    imagePrefix: platform-ingest-logstash-ubuntu-2204-aarch64
+    instanceType: "m6g.4xlarge"
+    diskSizeGb: 200
+  command: |
+    export WORKFLOW_TYPE="{workflow_type}"
+    export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+    export ARCH="aarch64"
+    eval "$(rbenv init -)"
+    .buildkite/scripts/dra/build_docker.sh
+'''
+
+    return step
+
+def publish_dra_step(branch, workflow_type, depends_on):
+    step = f'''
+- label: ":elastic-stack: Publish  / {branch}-{workflow_type.upper()} DRA artifacts"
+  key: "logstash_publish_dra"
+  depends_on: "{depends_on}"
+  agents:
+    provider: gcp
+    imageProject: elastic-images-prod
+    image: family/platform-ingest-logstash-ubuntu-2204
+    machineType: "n2-standard-16"
+    diskSizeGb: 200
+  command: |
+    echo "+++ Restoring Artifacts"
+    buildkite-agent artifact download "build/logstash*" .
+    buildkite-agent artifact download "build/distributions/**/*" .
+    echo "+++ Changing permissions for the release manager"
+    sudo chown -R :1000 build
+    echo "+++ Running DRA publish step"
+    export WORKFLOW_TYPE="{workflow_type}"
+    .buildkite/scripts/dra/publish.sh
+    '''
+
+    return step
+
+def build_steps_to_yaml(branch, workflow_type):
+    steps = []
+    steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type)))
+    steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type)))
+    steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type)))
+
+    return steps
+
+if __name__ == "__main__":
+    try:
+        workflow_type = os.environ["WORKFLOW_TYPE"]
+    except ImportError:
+        print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=<staging|snapshot>\n.Exiting.")
+        exit(1)
+
+    branch = os.environ["BUILDKITE_BRANCH"]
+
+    structure = {"steps": []}
+
+    # Group defining parallel steps that build and save artifacts
+    group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
+
+    structure["steps"].append({
+        "group": f":Build Artifacts - {workflow_type.upper()}",
+        "key": group_key,
+        "steps": build_steps_to_yaml(branch, workflow_type),
+    })
+
+    # Final step: pull artifacts built above and publish them via the release-manager
+    structure["steps"].extend(
+        yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
+    )
+
+    print(yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))