[ci] Commit DRA -SNAPSHOT buildkite pipeline #15337
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,36 @@ | ||
| # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json | ||
|
|
||
| ## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml) | ||
| steps: | ||
| - input: "Build parameters" | ||
| if: build.source != "schedule" | ||
| fields: | ||
| - text: "VERSION_QUALIFIER_OPT" | ||
| key: "VERSION_QUALIFIER_OPT" | ||
| default: "" | ||
| required: false | ||
| hint: "Optional version qualifier for built artifacts e.g.: alpha1,beta1" | ||
| - select: "DRA DRY-RUN" | ||
| key: "DRA_DRY_RUN" | ||
| required: false | ||
| default: "" | ||
| options: | ||
| - label: "True" | ||
| value: "--dry-run" | ||
| - label: "False" | ||
| value: "" | ||
| hint: "Whether the DRA release manager will actually publish artifacts, or run in dry-run mode." | ||
|
|
||
| - wait: ~ | ||
| if: build.source != "schedule" | ||
|
|
||
| - label: ":pipeline: Generate steps" | ||
| command: | | ||
| set -eo pipefail | ||
|
|
||
| export WORKFLOW_TYPE="snapshot" | ||
| python3 -m pip install pyyaml | ||
| echo "--- Generating dynamic pipeline steps:" | ||
| python3 .buildkite/scripts/dra/generatesteps.py | ||
|
|
||
| python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,90 @@ | ||
| #!/bin/bash -ie | ||
| #Note - ensure that the -e flag is set to properly set the $? status if any command fails | ||
| echo "####################################################################" | ||
| echo "##################### Starting $0" | ||
| echo "####################################################################" | ||
|
|
||
| source ./$(dirname "$0")/common.sh | ||
|
|
||
| # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values | ||
| case "$WORKFLOW_TYPE" in | ||
| snapshot) | ||
| info "Building artifacts for the $WORKFLOW_TYPE workflow..." | ||
| if [ -z "$VERSION_QUALIFIER_OPT" ]; then | ||
| rake artifact:docker || error "artifact:docker build failed." | ||
| rake artifact:docker_oss || error "artifact:docker_oss build failed." | ||
| rake artifact:dockerfiles || error "artifact:dockerfiles build failed." | ||
| if [ "$ARCH" != "aarch64" ]; then | ||
| rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." | ||
| fi | ||
| else | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed." | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed." | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed." | ||
| if [ "$ARCH" != "aarch64" ]; then | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." | ||
| fi | ||
| # Qualifier is passed from CI as optional field and specify the version postfix | ||
| # in case of alpha or beta releases: | ||
| # e.g: 8.0.0-alpha1 | ||
| STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" | ||
| fi | ||
| STACK_VERSION=${STACK_VERSION}-SNAPSHOT | ||
| info "Build complete, setting STACK_VERSION to $STACK_VERSION." | ||
| ;; | ||
| staging) | ||
| info "Building artifacts for the $WORKFLOW_TYPE workflow..." | ||
| if [ -z "$VERSION_QUALIFIER_OPT" ]; then | ||
|
There was a problem hiding this comment. I might not understand the code correctly and this could be a future improvement but, but this could be a function like There was a problem hiding this comment. Most of the bash scripts under |
||
| RELEASE=1 rake artifact:docker || error "artifact:docker build failed." | ||
| RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." | ||
| RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." | ||
| if [ "$ARCH" != "aarch64" ]; then | ||
| RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." | ||
| fi | ||
| else | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed." | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." | ||
| if [ "$ARCH" != "aarch64" ]; then | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." | ||
| fi | ||
| # Qualifier is passed from CI as optional field and specify the version postfix | ||
| # in case of alpha or beta releases: | ||
| # e.g: 8.0.0-alpha1 | ||
| STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" | ||
| fi | ||
| info "Build complete, setting STACK_VERSION to $STACK_VERSION." | ||
| ;; | ||
| *) | ||
| error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." | ||
| ;; | ||
| esac | ||
|
|
||
| info "Saving tar.gz for docker images" | ||
| save_docker_tarballs "${ARCH}" "${STACK_VERSION}" | ||
|
|
||
| info "Generated Artifacts" | ||
| for file in build/logstash-*; do shasum $file;done | ||
|
|
||
| info "Uploading DRA artifacts in buildkite's artifact store ..." | ||
| # Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the build_packages.sh | ||
| images="logstash logstash-oss" | ||
| if [ "$ARCH" != "aarch64" ]; then | ||
| # No logstash-ubi8 for AARCH64 | ||
| images="logstash logstash-oss logstash-ubi8" | ||
| fi | ||
| for image in ${images}; do | ||
| buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz" | ||
| done | ||
|
|
||
| # Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be | ||
| # overwritten when building aarch64 (or viceversa). | ||
| if [ "$ARCH" != "aarch64" ]; then | ||
| for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do | ||
| buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz" | ||
| done | ||
| fi | ||
|
|
||
| echo "####################################################################" | ||
| echo "##################### Finishing $0" | ||
| echo "####################################################################" | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| #!/bin/bash -ie | ||
| #Note - ensure that the -e flag is set to properly set the $? status if any command fails | ||
| echo "####################################################################" | ||
| echo "##################### Starting $0" | ||
| echo "####################################################################" | ||
|
|
||
| source ./$(dirname "$0")/common.sh | ||
|
|
||
| # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values | ||
| case "$WORKFLOW_TYPE" in | ||
| snapshot) | ||
| info "Building artifacts for the $WORKFLOW_TYPE workflow..." | ||
| if [ -z "$VERSION_QUALIFIER_OPT" ]; then | ||
| SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." | ||
| else | ||
| # Qualifier is passed from CI as optional field and specify the version postfix | ||
| # in case of alpha or beta releases: | ||
| # e.g: 8.0.0-alpha1 | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." | ||
| STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" | ||
| fi | ||
| STACK_VERSION=${STACK_VERSION}-SNAPSHOT | ||
| info "Build complete, setting STACK_VERSION to $STACK_VERSION." | ||
| ;; | ||
| staging) | ||
| info "Building artifacts for the $WORKFLOW_TYPE workflow..." | ||
| if [ -z "$VERSION_QUALIFIER_OPT" ]; then | ||
| RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." | ||
| else | ||
| # Qualifier is passed from CI as optional field and specify the version postfix | ||
| # in case of alpha or beta releases: | ||
| # e.g: 8.0.0-alpha1 | ||
| VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." | ||
| STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" | ||
| fi | ||
| info "Build complete, setting STACK_VERSION to $STACK_VERSION." | ||
| ;; | ||
| *) | ||
| error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." | ||
| ;; | ||
| esac | ||
|
|
||
| info "Generated Artifacts" | ||
| for file in build/logstash-*; do shasum $file;done | ||
|
|
||
| info "Creating dependencies report for ${STACK_VERSION}" | ||
| mkdir -p build/distributions/dependencies-reports/ | ||
| bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv | ||
|
|
||
| info "Generated dependencies report" | ||
| shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv | ||
|
|
||
| info "Uploading DRA artifacts in buildkite's artifact store ..." | ||
| buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*" | ||
|
|
||
| echo "####################################################################" | ||
| echo "##################### Finishing $0" | ||
| echo "####################################################################" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| function info { | ||
|
|
||
| echo "--- INFO: $1" | ||
| } | ||
|
|
||
| function error { | ||
| echo "--- ERROR: $1" | ||
| exit 1 | ||
| } | ||
|
|
||
| function save_docker_tarballs { | ||
| local arch="${1:?architecture required}" | ||
| local version="${2:?stack-version required}" | ||
| local images="logstash logstash-oss" | ||
| if [ "${arch}" != "aarch64" ]; then | ||
| # No logstash-ubi8 for AARCH64 | ||
| images="logstash logstash-oss logstash-ubi8" | ||
| fi | ||
|
|
||
| for image in ${images}; do | ||
| tar_file="${image}-${version}-docker-image-${arch}.tar" | ||
| docker save -o "build/${tar_file}" \ | ||
| "docker.elastic.co/logstash/${image}:${version}" || \ | ||
| error "Unable to save tar file ${tar_file} for ${image} image." | ||
| # NOTE: if docker save exited with non-zero the error log already exited the script | ||
| gzip "build/${tar_file}" | ||
| done | ||
| } | ||
|
|
||
| # Since we are using the system jruby, we need to make sure our jvm process | ||
| # uses at least 1g of memory, If we don't do this we can get OOM issues when | ||
| # installing gems. See https://github.com/elastic/logstash/issues/5179 | ||
| export JRUBY_OPTS="-J-Xmx1g" | ||
|
|
||
| # Extract the version number from the version.yml file | ||
| # e.g.: 8.6.0 | ||
| # The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable | ||
| STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'` | ||
|
|
||
| info "Agent is running on architecture [$(uname -i)]" | ||
|
|
||
| export VERSION_QUALIFIER_OPT=$(buildkite-agent meta-data get VERSION_QUALIFIER_OPT --default "") | ||
| export DRA_DRY_RUN=$(buildkite-agent meta-data get DRA_DRY_RUN --default "") | ||
|
|
||
| if [[ ! -z $DRA_DRY_RUN && $BUILDKITE_STEP_KEY == "logstash_publish_dra" ]]; then | ||
| info "Release manager will run in dry-run mode [$DRA_DRY_RUN]" | ||
| fi | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| DOCKER_REGISTRY="docker.elastic.co" | ||
| DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" | ||
| CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" | ||
|
|
||
|
|
||
| function docker_login { | ||
| DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") | ||
| DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") | ||
| docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null | ||
| unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET | ||
| } | ||
|
|
||
| function release_manager_login { | ||
| DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) | ||
| VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') | ||
| VAULT_ROLE_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') | ||
| VAULT_SECRET_ID=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') | ||
| export VAULT_ADDR_SECRET VAULT_ROLE_ID VAULT_SECRET_ID | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -euo pipefail | ||
|
|
||
| # Unset all variables ending with _SECRET or _TOKEN | ||
| for var in $(printenv | sed 's;=.*;;' | sort); do | ||
| if [[ $var != "VAULT_ADDR" && ("$var" == *_SECRET || "$var" == *_TOKEN || "$var" == *VAULT* ) ]]; then | ||
| unset "$var" | ||
| fi | ||
| done | ||
|
|
||
| if command -v docker &>/dev/null; then | ||
| DOCKER_REGISTRY="docker.elastic.co" | ||
| docker logout $DOCKER_REGISTRY | ||
| fi |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should probably remove this, I left it for review purposes.