[ci] Commit DRA -SNAPSHOT buildkite pipeline (#15337)

This commit adds support for building + publishing DRA (-SNAPSHOT for now) artifacts for Logstash. It builds on top of #15312 and therefore only targets the `main` branch and is intended to be run manually during a trial period before we retire the corresponding Jenkins job. The structure is similar to Jenkins: 1. Three steps runs in parallel to build packages, x86_64 docker and aarch64 docker artifacts. 2. Once 1. is successfully done, use release manager to publish the artifacts. We generate the pipeline steps for 1. and 2. dynamically (with a simple Python script) to avoid repetition for future PRs: we will add a new pipeline in a follow up PR for -STAGING. The actual shell scripts are simplified copies from the existing `dra*` scripts under https://github.com/elastic/logstash/tree/main/ci; the simplification comes from native support for copying artifacts between steps in Buildkite and not having to use an intermediate bucket. Relates: elastic/ingest-dev#1720 Blocked by: https://github.com/elastic/ci/pull/2312/files
elastic · Sep 26, 2023 · dd77c9f · dd77c9f
1 parent b31c429
commit dd77c9f
Show file tree

Hide file tree

Showing 8 changed files with 500 additions and 9 deletions.
diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml
@@ -1,12 +1,36 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
-agents:
-  cpu: "2"
-  memory: "4Gi"
-  ephemeralStorage: "20Gi"
-
+## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml)
 steps:
-  - label: ":wave: Greetings"
-    command: "echo 'TODO: run DRA Logstash'"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.1"
+  - input: "Build parameters"
+    if: build.source != "schedule"
+    fields:
+      - text: "VERSION_QUALIFIER_OPT"
+        key: "VERSION_QUALIFIER_OPT"
+        default: ""
+        required: false
+        hint: "Optional version qualifier for built artifacts e.g.: alpha1,beta1"
+      - select: "DRA DRY-RUN"
+        key: "DRA_DRY_RUN"
+        required: false
+        default: ""
+        options:
+          - label: "True"
+            value: "--dry-run"
+          - label: "False"
+            value: ""
+        hint: "Whether the DRA release manager will actually publish artifacts, or run in dry-run mode."
+
+  - wait: ~
+    if: build.source != "schedule"
+
+  - label: ":pipeline: Generate steps"
+    command: |
+      set -eo pipefail
+
+      export WORKFLOW_TYPE="snapshot"
+      python3 -m pip install pyyaml
+      echo "--- Generating dynamic pipeline steps:"
+      python3 .buildkite/scripts/dra/generatesteps.py
+      
+      python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload
diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh
@@ -0,0 +1,90 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/common.sh
+
+# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            rake artifact:docker || error "artifact:docker build failed."
+            rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+        else
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    staging)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
+            RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+        else
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+            if [ "$ARCH" != "aarch64" ]; then
+                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+            fi
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    *)
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Saving tar.gz for docker images"
+save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
+
+info "Generated Artifacts"
+for file in build/logstash-*; do shasum $file;done
+
+info "Uploading DRA artifacts in buildkite's artifact store ..."
+# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh
+images="logstash logstash-oss"
+if [ "$ARCH" != "aarch64" ]; then
+    # No logstash-ubi8 for AARCH64
+    images="logstash logstash-oss logstash-ubi8"
+fi
+for image in ${images}; do
+    buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz"
+done
+
+# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be
+# overwritten when building aarch64 (or viceversa).
+if [ "$ARCH" != "aarch64" ]; then
+    for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do
+        buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz"
+    done
+fi
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"
diff --git a/.buildkite/scripts/dra/build_packages.sh b/.buildkite/scripts/dra/build_packages.sh
@@ -0,0 +1,58 @@
+#!/bin/bash -ie
+#Note - ensure that the -e flag is set to properly set the $? status if any command fails
+echo "####################################################################"
+echo "##################### Starting $0"
+echo "####################################################################"
+
+source ./$(dirname "$0")/common.sh
+
+# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+case "$WORKFLOW_TYPE" in
+    snapshot)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+        else
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    staging)
+        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
+        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
+            RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+        else
+            # Qualifier is passed from CI as optional field and specify the version postfix
+            # in case of alpha or beta releases:
+            # e.g: 8.0.0-alpha1
+            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
+        fi
+        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        ;;
+    *)
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        ;;
+esac
+
+info "Generated Artifacts"
+for file in build/logstash-*; do shasum $file;done
+
+info "Creating dependencies report for ${STACK_VERSION}"
+mkdir -p build/distributions/dependencies-reports/
+bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
+
+info "Generated dependencies report"
+shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv
+
+info "Uploading DRA artifacts in buildkite's artifact store ..."
+buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*"
+
+echo "####################################################################"
+echo "##################### Finishing $0"
+echo "####################################################################"
diff --git a/.buildkite/scripts/dra/common.sh b/.buildkite/scripts/dra/common.sh
@@ -0,0 +1,47 @@
+function info {
+    echo "--- INFO: $1"
+}
+
+function error {
+    echo "--- ERROR: $1"
+    exit 1
+}
+
+function save_docker_tarballs {
+    local arch="${1:?architecture required}"
+    local version="${2:?stack-version required}"
+    local images="logstash logstash-oss"
+    if [ "${arch}" != "aarch64" ]; then
+        # No logstash-ubi8 for AARCH64
+        images="logstash logstash-oss logstash-ubi8"
+    fi
+
+    for image in ${images}; do
+        tar_file="${image}-${version}-docker-image-${arch}.tar"
+        docker save -o "build/${tar_file}" \
+            "docker.elastic.co/logstash/${image}:${version}" || \
+            error "Unable to save tar file ${tar_file} for ${image} image."
+        # NOTE: if docker save exited with non-zero the error log already exited the script
+        gzip "build/${tar_file}"
+    done
+}
+
+# Since we are using the system jruby, we need to make sure our jvm process
+# uses at least 1g of memory, If we don't do this we can get OOM issues when
+# installing gems. See https://github.com/elastic/logstash/issues/5179
+export JRUBY_OPTS="-J-Xmx1g"
+
+# Extract the version number from the version.yml file
+# e.g.: 8.6.0
+# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable
+STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'`
+
+info "Agent is running on architecture [$(uname -i)]"
+
+export VERSION_QUALIFIER_OPT=$(buildkite-agent meta-data get VERSION_QUALIFIER_OPT --default "")
+export DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "")
+
+if [[ ! -z $DRA_DRY_RUN && $BUILDKITE_STEP_KEY == "logstash_publish_dra" ]]; then
+    info "Release manager will run in dry-run mode [$DRA_DRY_RUN]"
+fi
+
diff --git a/.buildkite/scripts/dra/docker-env-setup.sh b/.buildkite/scripts/dra/docker-env-setup.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -euo pipefail
+
+DOCKER_REGISTRY="docker.elastic.co"
+DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
+CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
+
+
+function docker_login {
+  DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
+  DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
+  docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
+  unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET
+}
+
+function release_manager_login {
+  DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
+  VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
+  VAULT_ROLE_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')
+  VAULT_SECRET_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
+  export VAULT_ADDR_SECRET VAULT_ROLE_ID VAULT_SECRET_ID
+}
diff --git a/.buildkite/scripts/dra/docker-env-teardown.sh b/.buildkite/scripts/dra/docker-env-teardown.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Unset all variables ending with _SECRET or _TOKEN
+for var in $(printenv | sed 's;=.*;;' | sort); do
+  if [[ $var != "VAULT_ADDR" && ("$var" == *_SECRET || "$var" == *_TOKEN || "$var" == *VAULT* ) ]]; then
+      unset "$var"
+  fi
+done
+
+if command -v docker &>/dev/null; then
+  DOCKER_REGISTRY="docker.elastic.co"
+  docker logout $DOCKER_REGISTRY
+fi