diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml index 988848fe88e..a7821022d70 100644 --- a/.buildkite/dra_pipeline.yml +++ b/.buildkite/dra_pipeline.yml @@ -1,12 +1,36 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json -agents: - cpu: "2" - memory: "4Gi" - ephemeralStorage: "20Gi" - +## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml) steps: - - label: ":wave: Greetings" - command: "echo 'TODO: run DRA Logstash'" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.1" + - input: "Build parameters" + if: build.source != "schedule" + fields: + - text: "VERSION_QUALIFIER_OPT" + key: "VERSION_QUALIFIER_OPT" + default: "" + required: false + hint: "Optional version qualifier for built artifacts e.g.: alpha1,beta1" + - select: "DRA DRY-RUN" + key: "DRA_DRY_RUN" + required: false + default: "" + options: + - label: "True" + value: "--dry-run" + - label: "False" + value: "" + hint: "Whether the DRA release manager will actually publish artifacts, or run in dry-run mode." + + - wait: ~ + if: build.source != "schedule" + + - label: ":pipeline: Generate steps" + command: | + set -eo pipefail + + export WORKFLOW_TYPE="snapshot" + python3 -m pip install pyyaml + echo "--- Generating dynamic pipeline steps:" + python3 .buildkite/scripts/dra/generatesteps.py + + python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh new file mode 100755 index 00000000000..fee8cce0eea --- /dev/null +++ b/.buildkite/scripts/dra/build_docker.sh @@ -0,0 +1,90 @@ +#!/bin/bash -ie +#Note - ensure that the -e flag is set to properly set the $? status if any command fails +echo "####################################################################" +echo "##################### Starting $0" +echo "####################################################################" + +source ./$(dirname "$0")/common.sh + +# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values +case "$WORKFLOW_TYPE" in + snapshot) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + rake artifact:docker || error "artifact:docker build failed." + rake artifact:docker_oss || error "artifact:docker_oss build failed." + rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + else + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + STACK_VERSION=${STACK_VERSION}-SNAPSHOT + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + staging) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + RELEASE=1 rake artifact:docker || error "artifact:docker build failed." + RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." + RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + else + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + *) + error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." + ;; +esac + +info "Saving tar.gz for docker images" +save_docker_tarballs "${ARCH}" "${STACK_VERSION}" + +info "Generated Artifacts" +for file in build/logstash-*; do shasum $file;done + +info "Uploading DRA artifacts in buildkite's artifact store ..." +# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh +images="logstash logstash-oss" +if [ "$ARCH" != "aarch64" ]; then + # No logstash-ubi8 for AARCH64 + images="logstash logstash-oss logstash-ubi8" +fi +for image in ${images}; do + buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz" +done + +# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be +# overwritten when building aarch64 (or viceversa). +if [ "$ARCH" != "aarch64" ]; then + for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do + buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz" + done +fi + +echo "####################################################################" +echo "##################### Finishing $0" +echo "####################################################################" diff --git a/.buildkite/scripts/dra/build_packages.sh b/.buildkite/scripts/dra/build_packages.sh new file mode 100755 index 00000000000..2bd4ed1750a --- /dev/null +++ b/.buildkite/scripts/dra/build_packages.sh @@ -0,0 +1,58 @@ +#!/bin/bash -ie +#Note - ensure that the -e flag is set to properly set the $? status if any command fails +echo "####################################################################" +echo "##################### Starting $0" +echo "####################################################################" + +source ./$(dirname "$0")/common.sh + +# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values +case "$WORKFLOW_TYPE" in + snapshot) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + else + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + STACK_VERSION=${STACK_VERSION}-SNAPSHOT + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + staging) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + else + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + *) + error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." + ;; +esac + +info "Generated Artifacts" +for file in build/logstash-*; do shasum $file;done + +info "Creating dependencies report for ${STACK_VERSION}" +mkdir -p build/distributions/dependencies-reports/ +bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv + +info "Generated dependencies report" +shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv + +info "Uploading DRA artifacts in buildkite's artifact store ..." +buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*" + +echo "####################################################################" +echo "##################### Finishing $0" +echo "####################################################################" diff --git a/.buildkite/scripts/dra/common.sh b/.buildkite/scripts/dra/common.sh new file mode 100755 index 00000000000..c398322a158 --- /dev/null +++ b/.buildkite/scripts/dra/common.sh @@ -0,0 +1,47 @@ +function info { + echo "--- INFO: $1" +} + +function error { + echo "--- ERROR: $1" + exit 1 +} + +function save_docker_tarballs { + local arch="${1:?architecture required}" + local version="${2:?stack-version required}" + local images="logstash logstash-oss" + if [ "${arch}" != "aarch64" ]; then + # No logstash-ubi8 for AARCH64 + images="logstash logstash-oss logstash-ubi8" + fi + + for image in ${images}; do + tar_file="${image}-${version}-docker-image-${arch}.tar" + docker save -o "build/${tar_file}" \ + "docker.elastic.co/logstash/${image}:${version}" || \ + error "Unable to save tar file ${tar_file} for ${image} image." + # NOTE: if docker save exited with non-zero the error log already exited the script + gzip "build/${tar_file}" + done +} + +# Since we are using the system jruby, we need to make sure our jvm process +# uses at least 1g of memory, If we don't do this we can get OOM issues when +# installing gems. See https://github.com/elastic/logstash/issues/5179 +export JRUBY_OPTS="-J-Xmx1g" + +# Extract the version number from the version.yml file +# e.g.: 8.6.0 +# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable +STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'` + +info "Agent is running on architecture [$(uname -i)]" + +export VERSION_QUALIFIER_OPT=$(buildkite-agent meta-data get VERSION_QUALIFIER_OPT --default "") +export DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "") + +if [[ ! -z $DRA_DRY_RUN && $BUILDKITE_STEP_KEY == "logstash_publish_dra" ]]; then + info "Release manager will run in dry-run mode [$DRA_DRY_RUN]" +fi + diff --git a/.buildkite/scripts/dra/docker-env-setup.sh b/.buildkite/scripts/dra/docker-env-setup.sh new file mode 100755 index 00000000000..6787a4e9dbe --- /dev/null +++ b/.buildkite/scripts/dra/docker-env-setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euo pipefail + +DOCKER_REGISTRY="docker.elastic.co" +DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + + +function docker_login { + DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null + unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET +} + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID VAULT_SECRET_ID +} diff --git a/.buildkite/scripts/dra/docker-env-teardown.sh b/.buildkite/scripts/dra/docker-env-teardown.sh new file mode 100755 index 00000000000..8e7fe0a9a64 --- /dev/null +++ b/.buildkite/scripts/dra/docker-env-teardown.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +# Unset all variables ending with _SECRET or _TOKEN +for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ $var != "VAULT_ADDR" && ("$var" == *_SECRET || "$var" == *_TOKEN || "$var" == *VAULT* ) ]]; then + unset "$var" + fi +done + +if command -v docker &>/dev/null; then + DOCKER_REGISTRY="docker.elastic.co" + docker logout $DOCKER_REGISTRY +fi diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py new file mode 100644 index 00000000000..d59b1350102 --- /dev/null +++ b/.buildkite/scripts/dra/generatesteps.py @@ -0,0 +1,134 @@ +import os +import sys + +import yaml + +def to_bk_key_friendly_string(key): + """ + Convert and return key to an acceptable format for Buildkite's key: field + Only alphanumerics, dashes and underscores are allowed. + """ + + mapping_table = str.maketrans({'.': '_'}) + + return key.translate(mapping_table) + +def package_x86_step(branch, workflow_type): + step = f''' +- label: ":package: Build packages / {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_build_packages_dra" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.2" + cpu: "8" + memory: "16Gi" + ephemeralStorage: "200Gi" + command: | + export WORKFLOW_TYPE="{workflow_type}" + export PATH="/usr/local/rbenv/bin:$PATH" + eval "$(rbenv init -)" + .buildkite/scripts/dra/build_packages.sh +''' + + return step + +def package_x86_docker_step(branch, workflow_type): + step = f''' +- label: ":package: Build x86_64 Docker / {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_build_x86_64_docker_dra" + agents: + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-16" + diskSizeGb: 200 + command: | + export WORKFLOW_TYPE="{workflow_type}" + export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" + export ARCH="x86_64" + eval "$(rbenv init -)" + .buildkite/scripts/dra/build_docker.sh +''' + + return step + +def package_aarch64_docker_step(branch, workflow_type): + step = f''' +- label: ":package: Build aarch64 Docker / {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_build_aarch64_docker_dra" + agents: + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-logstash-ubuntu-2204-aarch64 + machineType: "t2a-standard-8" + diskSizeGb: 200 + region: 'us-central1' + # so far only these regions support t2a instance types + zones: "us-central1-a,us-central1-b,us-central1-f" + command: | + export WORKFLOW_TYPE="{workflow_type}" + export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" + export ARCH="aarch64" + eval "$(rbenv init -)" + .buildkite/scripts/dra/build_docker.sh +''' + + return step + +def publish_dra_step(branch, workflow_type, depends_on): + step = f''' +- label: ":elastic-stack: Publish / {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_publish_dra" + depends_on: "{depends_on}" + agents: + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-16" + diskSizeGb: 200 + command: | + echo "+++ Restoring Artifacts" + buildkite-agent artifact download "build/logstash*" . + buildkite-agent artifact download "build/distributions/**/*" . + echo "+++ Changing permissions for the release manager" + sudo chown -R :1000 build + echo "+++ Running DRA publish step" + export WORKFLOW_TYPE="{workflow_type}" + .buildkite/scripts/dra/publish.sh + ''' + + return step + +def build_steps_to_yaml(branch, workflow_type): + steps = [] + steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type))) + + return steps + +if __name__ == "__main__": + try: + workflow_type = os.environ["WORKFLOW_TYPE"] + except ImportError: + print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=\n.Exiting.") + exit(1) + + branch = os.environ["BUILDKITE_BRANCH"] + + structure = {"steps": []} + + # Group defining parallel steps that build and save artifacts + group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}") + + structure["steps"].append({ + "group": f":Build Artifacts - {workflow_type.upper()}", + "key": group_key, + "steps": build_steps_to_yaml(branch, workflow_type), + }) + + # Final step: pull artifacts built above and publish them via the release-manager + structure["steps"].extend( + yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)), + ) + + print(yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False)) diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh new file mode 100755 index 00000000000..f501b05a1d3 --- /dev/null +++ b/.buildkite/scripts/dra/publish.sh @@ -0,0 +1,100 @@ +#!/bin/bash -i +set -euo pipefail + +echo "####################################################################" +echo "##################### Starting $0" +echo "####################################################################" + +source ./$(dirname "$0")/common.sh + +PLAIN_STACK_VERSION=$STACK_VERSION + +# This is the branch selector that needs to be passed to the release-manager +# It has to be the name of the branch which originates the artifacts. +RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p'` +if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then + RELEASE_BRANCH=$RELEASE_VER +else + RELEASE_BRANCH=main +fi + +if [ -n "$VERSION_QUALIFIER_OPT" ]; then + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}" +fi + +case "$WORKFLOW_TYPE" in + snapshot) + STACK_VERSION=${STACK_VERSION}-SNAPSHOT + ;; + staging) + ;; + *) + error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..." + ;; +esac + +info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}" + +if [ "$RELEASE_VER" != "7.17" ]; then + # Version 7.17.x doesn't generates ARM artifacts for Darwin + # TODO see if we need to do anything here + : +fi + +# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected +# by the release manager. +# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle +# for more details. +# TODO filter it out when uploading artifacts instead +rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz + +info "Downloaded ARTIFACTS sha report" +for file in build/logstash-*; do shasum $file;done + +mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv + +# set required permissions on artifacts and directory +chmod -R a+r build/* +chmod -R a+w build + +chmod -R a+r $PWD/* +chmod -R a+w $PWD + +info "Setup docker credentials" +# TODO disable tracing +# set +o xtrace +source ./$(dirname "$0")/docker-env-setup.sh +release_manager_login + +# ensure the latest image has been pulled +docker pull docker.elastic.co/infra/release-manager:latest + +info "Running the release manager ..." + +# collect the artifacts for use with the unified build +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID \ + -e VAULT_SECRET_ID \ + --mount type=bind,readonly=false,src="$PWD",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project logstash \ + --branch ${RELEASE_BRANCH} \ + --commit "$(git rev-parse HEAD)" \ + --workflow "${WORKFLOW_TYPE}" \ + --version "${PLAIN_STACK_VERSION}" \ + --artifact-set main \ + ${DRA_DRY_RUN} + +info "Teardown logins" +$(dirname "$0")/docker-env-teardown.sh + +echo "####################################################################" +echo "##################### Finishing $0" +echo "####################################################################"