diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml index 988848fe88e..1f747248bb3 100644 --- a/.buildkite/dra_pipeline.yml +++ b/.buildkite/dra_pipeline.yml @@ -1,12 +1,26 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json -agents: - cpu: "2" - memory: "4Gi" - ephemeralStorage: "20Gi" - +## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml) steps: - - label: ":wave: Greetings" - command: "echo 'TODO: run DRA Logstash'" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.1" + - input: "Build parameters" + if: build.source != "schedule" + fields: + - text: "VERSION_QUALIFIER_OPT" + key: "VERSION_QUALIFIER_OPT" + default: "" + required: false + hint: "Optional version qualifier for built artifacts e.g.: alpha1,beta1" + + - wait: ~ + if: build.source != "schedule" + + - label: ":pipeline: Generate steps" + command: | + set -eo pipefail + + export WORKFLOW_TYPE="snapshot" + python3 -m pip install pyyaml + echo "--- Generating dynamic pipeline steps:" + python3 .buildkite/scripts/dra/generatesteps.py + + python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 00000000000..ee86f6b3093 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,39 @@ +#!/bin/bash + +set -euo pipefail + +DOCKER_REGISTRY="docker.elastic.co" +DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + + +function docker_login { + DOCKER_USERNAME_SECRET=$(retry -t 5 -- vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}") + DOCKER_PASSWORD_SECRET=$(retry -t 5 -- vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}") + docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null + unset DOCKER_USERNAME_SECRET DOCKER_PASSWORD_SECRET +} + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "logstash-dra-pipeline-ci" ]]; then + + vq=$(buildkite-agent meta-data get VERSION_QUALIFIER_OPT --default "") + if [[ ! -z "$vq" ]]; then + echo "+++ Exporting VERSION_QUALIFIER_OPT to value [$vq]" + export VERSION_QUALIFIER_OPT=$vq + else + echo "+++ No value provided for VERSION_QUALIFIER_OPT leaving unset" + fi + + if [[ "$BUILDKITE_STEP_KEY" == "logstash_publish_dra" ]]; then + echo "+++ Setting DRA params" + release_manager_login + fi +fi diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100755 index 00000000000..13343350b24 --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,10 @@ +#!/bin/bash + +set -eo pipefail + +source .buildkite/scripts/unset-secrets.sh + +if command -v docker &>/dev/null; then + DOCKER_REGISTRY="docker.elastic.co" + docker logout $DOCKER_REGISTRY +fi diff --git a/.buildkite/scripts/dra/dra_common.sh b/.buildkite/scripts/dra/dra_common.sh new file mode 100755 index 00000000000..c27453ccd02 --- /dev/null +++ b/.buildkite/scripts/dra/dra_common.sh @@ -0,0 +1,39 @@ +function info { + echo "--- INFO: $1" +} + +function error { + echo "--- ERROR: $1" + exit 1 +} + +function save_docker_tarballs { + local arch="${1:?architecture required}" + local version="${2:?stack-version required}" + local images="logstash logstash-oss" + if [ "${arch}" != "aarch64" ]; then + # No logstash-ubi8 for AARCH64 + images="logstash logstash-oss logstash-ubi8" + fi + + for image in ${images}; do + tar_file="${image}-${version}-docker-image-${arch}.tar" + docker save -o "build/${tar_file}" \ + "docker.elastic.co/logstash/${image}:${version}" || \ + error "Unable to save tar file ${tar_file} for ${image} image." + # NOTE: if docker save exited with non-zero the error log already exited the script + gzip "build/${tar_file}" + done +} + +# Since we are using the system jruby, we need to make sure our jvm process +# uses at least 1g of memory, If we don't do this we can get OOM issues when +# installing gems. See https://github.com/elastic/logstash/issues/5179 +export JRUBY_OPTS="-J-Xmx1g" + +# Extract the version number from the version.yml file +# e.g.: 8.6.0 +# The suffix part like alpha1 etc is managed by the optional VERSION_QUALIFIER_OPT environment variable +STACK_VERSION=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\)$/\1/p'` + +info "Agent is running on architecture [$(uname -i)]" diff --git a/.buildkite/scripts/dra/dra_docker.sh b/.buildkite/scripts/dra/dra_docker.sh new file mode 100755 index 00000000000..5099c80da4c --- /dev/null +++ b/.buildkite/scripts/dra/dra_docker.sh @@ -0,0 +1,90 @@ +#!/bin/bash -ie +#Note - ensure that the -e flag is set to properly set the $? status if any command fails +echo "####################################################################" +echo "##################### Starting $0" +echo "####################################################################" + +source ./$(dirname "$0")/dra_common.sh + +# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values +case "$WORKFLOW_TYPE" in + snapshot) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + rake artifact:docker || error "artifact:docker build failed." + rake artifact:docker_oss || error "artifact:docker_oss build failed." + rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + else + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + STACK_VERSION=${STACK_VERSION}-SNAPSHOT + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + staging) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + RELEASE=1 rake artifact:docker || error "artifact:docker build failed." + RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." + RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + else + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed." + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed." + if [ "$ARCH" != "aarch64" ]; then + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed." + fi + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + *) + error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." + ;; +esac + +info "Saving tar.gz for docker images" +save_docker_tarballs "${ARCH}" "${STACK_VERSION}" + +info "Generated Artifacts" +for file in build/logstash-*; do shasum $file;done + +info "Uploading DRA artifacts in buildkite's artifact store ..." +# Note the deb, rpm tar.gz AARCH64 files generated has already been loaded by the dra_x86_64.sh +images="logstash logstash-oss" +if [ "$ARCH" != "aarch64" ]; then + # No logstash-ubi8 for AARCH64 + images="logstash logstash-oss logstash-ubi8" +fi +for image in ${images}; do + buildkite-agent artifact upload "build/$image-${STACK_VERSION}-docker-image-${ARCH}.tar.gz" +done + +# Upload 'docker-build-context.tar.gz' files only when build x86_64, otherwise they will be +# overwritten when building aarch64 (or viceversa). +if [ "$ARCH" != "aarch64" ]; then + for image in logstash logstash-oss logstash-ubi8 logstash-ironbank; do + buildkite-agent artifact upload "build/${image}-${STACK_VERSION}-docker-build-context.tar.gz" + done +fi + +echo "####################################################################" +echo "##################### Finishing $0" +echo "####################################################################" diff --git a/.buildkite/scripts/dra/dra_upload.sh b/.buildkite/scripts/dra/dra_upload.sh new file mode 100755 index 00000000000..f3820c41cca --- /dev/null +++ b/.buildkite/scripts/dra/dra_upload.sh @@ -0,0 +1,85 @@ +#!/bin/bash -iex +#Note - ensure that the -e flag is set to properly set the $? status if any command fails +echo "####################################################################" +echo "##################### Starting $0" +echo "####################################################################" + +source ./$(dirname "$0")/dra_common.sh +PLAIN_STACK_VERSION=$STACK_VERSION + +# This is the branch selector that needs to be passed to the release-manager +# It has to be the name of the branch which originates the artifacts. +RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p'` +if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then + RELEASE_BRANCH=$RELEASE_VER +else + RELEASE_BRANCH=main +fi + +if [ -n "$VERSION_QUALIFIER_OPT" ]; then + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}" +fi + +case "$WORKFLOW_TYPE" in + snapshot) + STACK_VERSION=${STACK_VERSION}-SNAPSHOT + ;; + staging) + ;; + *) + error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..." + ;; +esac + +info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}" + +if [ "$RELEASE_VER" != "7.17" ]; then + # Version 7.17.x doesn't generates ARM artifacts for Darwin + # TODO see if we need to do anything here +fi + +# Deleting ubi8 for aarch64 for the time being. This image itself is not being built, and it is not expected +# by the release manager. +# See https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/8.5/logstash.gradle +# for more details. +# TODO filter it out when uploading artifacts instead +rm build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz + +info "Downloaded ARTIFACTS sha report" +for file in build/logstash-*; do shasum $file;done + +mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv + +# set required permissions on artifacts and directory +chmod -R a+r build/* +chmod -R a+w build + +chmod -R a+r $PWD/* +chmod -R a+w $PWD + +# ensure the latest image has been pulled +docker pull docker.elastic.co/infra/release-manager:latest + +# collect the artifacts for use with the unified build +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="$PWD",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project logstash \ + --branch ${RELEASE_BRANCH} \ + --commit "$(git rev-parse HEAD)" \ + --workflow "${WORKFLOW_TYPE}" \ + --version "${PLAIN_STACK_VERSION}" \ + --artifact-set main + +echo "####################################################################" +echo "##################### Finishing $0" +echo "####################################################################" diff --git a/.buildkite/scripts/dra/dra_x86_64.sh b/.buildkite/scripts/dra/dra_x86_64.sh new file mode 100755 index 00000000000..ba073f4d819 --- /dev/null +++ b/.buildkite/scripts/dra/dra_x86_64.sh @@ -0,0 +1,58 @@ +#!/bin/bash -ie +#Note - ensure that the -e flag is set to properly set the $? status if any command fails +echo "####################################################################" +echo "##################### Starting $0" +echo "####################################################################" + +source ./$(dirname "$0")/dra_common.sh + +# WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values +case "$WORKFLOW_TYPE" in + snapshot) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + else + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + STACK_VERSION=${STACK_VERSION}-SNAPSHOT + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + staging) + info "Building artifacts for the $WORKFLOW_TYPE workflow..." + if [ -z "$VERSION_QUALIFIER_OPT" ]; then + RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + else + # Qualifier is passed from CI as optional field and specify the version postfix + # in case of alpha or beta releases: + # e.g: 8.0.0-alpha1 + VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed." + STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}" + fi + info "Build complete, setting STACK_VERSION to $STACK_VERSION." + ;; + *) + error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..." + ;; +esac + +info "Generated Artifacts" +for file in build/logstash-*; do shasum $file;done + +info "Creating dependencies report for ${STACK_VERSION}" +mkdir -p build/distributions/dependencies-reports/ +bin/dependencies-report --csv=build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv + +info "Generated dependencies report" +shasum build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv + +info "Uploading DRA artifacts in buildkite's artifact store ..." +buildkite-agent artifact upload "build/logstash*;build/distributions/dependencies-reports/logstash*" + +echo "####################################################################" +echo "##################### Finishing $0" +echo "####################################################################" diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py new file mode 100644 index 00000000000..6e1db1c7f6e --- /dev/null +++ b/.buildkite/scripts/dra/generatesteps.py @@ -0,0 +1,131 @@ +import os +import sys + +import yaml + +def to_bk_key_friendly_string(key): + """ + Convert and return key to an acceptable format for Buildkite's key: field + Only alphanumerics, dashes and underscores are allowed. + """ + + mapping_table = str.maketrans({'.': '_'}) + + return key.translate(mapping_table) + +def package_x86_step(branch, workflow_type): + step = f''' +- label: ":package: Build x86_64 {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_build_x86_64_dra" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.2" + cpu: "8" + memory: "16Gi" + ephemeralStorage: "200Gi" + command: | + export WORKFLOW_TYPE={workflow_type} + export PATH="/usr/local/rbenv/bin:$PATH" + eval "$(rbenv init -)" + .buildkite/scripts/dra/dra_x86_64.sh +''' + + return step + +def package_x86_docker_step(branch, workflow_type): + step = f''' +- label: ":package: Build x86_64 Docker {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_build_x86_64_docker_dra" + agents: + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-16" + diskSizeGb: 200 + command: | + export WORKFLOW_TYPE={workflow_type} + export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" + export ARCH="x86_64" + eval "$(rbenv init -)" + .buildkite/scripts/dra/dra_docker.sh +''' + + return step + +def package_aarch64_docker_step(branch, workflow_type): + step = f''' +- label: ":package: Build aarch64 Docker {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_build_aarch64_docker_dra" + agents: + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-logstash-ubuntu-2204-aarch64 + machineType: "t2a-standard-8" + diskSizeGb: 200 + command: | + export WORKFLOW_TYPE={workflow_type} + export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH" + export ARCH="aarch64" + eval "$(rbenv init -)" + .buildkite/scripts/dra/dra_docker.sh +''' + + return step + +def publish_dra_step(branch, workflow_type, depends_on): + step = f''' +- label: ":elastic-stack: Publish {branch}-{workflow_type.upper()} DRA artifacts" + key: "logstash_publish_dra" + depends_on: "{depends_on}" + agents: + provider: gcp + imageProject: elastic-images-qa + image: family/platform-ingest-logstash-ubuntu-2204 + machineType: "n2-standard-16" + diskSizeGb: 200 + command: | + echo "+++ Restoring Artifacts" + buildkite-agent artifact download "build/logstash*" . + buildkite-agent artifact download "build/distributions/**/*" . + echo "+++ Changing permissions for the release manager" + sudo chown -R :1000 build + echo "+++ Running DRA publish step" + ls -laRt build + .buildkite/scripts/dra/dra_upload.sh + ''' + + return step + +def build_steps_to_yaml(branch, workflow_type): + steps = [] + steps.extend(yaml.safe_load(package_x86_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_x86_docker_step(branch, workflow_type))) + steps.extend(yaml.safe_load(package_aarch64_docker_step(branch, workflow_type))) + + return steps + +if __name__ == "__main__": + try: + workflow_type = os.environ["WORKFLOW_TYPE"] + except ImportError: + print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=\n.Exiting.") + exit(1) + + branch = os.environ["BUILDKITE_BRANCH"] + + structure = {"steps": []} + + # Group defining parallel steps that build and save artifacts + group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}") + + structure["steps"].append({ + "group": f":Build Artifacts - {workflow_type.upper()}", + "key": group_key, + "steps": build_steps_to_yaml(branch, workflow_type), + }) + + # Final step: pull artifacts built above and publish them via the release-manager + structure["steps"].extend( + yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)), + ) + + print(yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False)) diff --git a/.buildkite/scripts/unset-secrets.sh b/.buildkite/scripts/unset-secrets.sh new file mode 100644 index 00000000000..88815257661 --- /dev/null +++ b/.buildkite/scripts/unset-secrets.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail +# Unset all variables ending with _SECRET or _TOKEN +for var in $(printenv | sed 's;=.*;;' | sort); do + if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then + unset "$var" + fi +done \ No newline at end of file diff --git a/build.gradle b/build.gradle index 6df2890e392..32d0cf30315 100644 --- a/build.gradle +++ b/build.gradle @@ -37,9 +37,10 @@ buildscript { plugins { id "de.undercouch.download" version "4.0.4" id "com.dorongold.task-tree" version "2.1.0" +// id "jacoco" +// id "org.sonarqube" version "4.3.0.3225" } - apply plugin: 'de.undercouch.download' apply from: "rubyUtils.gradle" @@ -57,7 +58,6 @@ allprojects { apply plugin: 'java' apply plugin: 'idea' apply plugin: 'java-library' - project.sourceCompatibility = JavaVersion.VERSION_11 project.targetCompatibility = JavaVersion.VERSION_11 @@ -860,7 +860,7 @@ if (System.getenv('OSS') != 'true') { dependsOn copyPluginTestAlias dependsOn ":logstash-xpack:rubyTests" } - tasks.register("runXPackIntegrationTests"){ + tasks.register("runXPackIntegrationTests") { dependsOn copyPluginTestAlias dependsOn ":logstash-xpack:rubyIntegrationTests" - } + } \ No newline at end of file diff --git a/ci/docker_run.sh b/ci/docker_run.sh index f3c088c1c4a..49b814e048c 100755 --- a/ci/docker_run.sh +++ b/ci/docker_run.sh @@ -1,10 +1,7 @@ #!/bin/bash # Syntax is docker_run.sh IMAGE_NAME SCRIPT_NAME *EXTRA_DOCKER_OPTS -set -x # We want verbosity here, this mostly runs on CI and we want to easily debug stuff - #Note - ensure that the -e flag is NOT set, and explicitly check the $? status to allow for clean up - REMOVE_IMAGE=false DOCKER_EXTERNAL_JDK="" if [ -z "$branch_specifier" ]; then @@ -16,7 +13,7 @@ else fi if [ "$OSS" == "true" ]; then - DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} --env OSS=true" + DOCKER_ENV_OPTS="${DOCKER_ENV_OPTS} -e OSS=true" fi echo "Running Docker CI build for '$IMAGE_NAME' " diff --git a/ci/docker_unit_tests.sh b/ci/docker_unit_tests.sh index c7557b5d836..5c33bb90de8 100755 --- a/ci/docker_unit_tests.sh +++ b/ci/docker_unit_tests.sh @@ -1,2 +1,10 @@ #!/bin/bash -ci/docker_run.sh logstash-unit-tests ci/unit_tests.sh $@ +# Init vault +VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID") +export VAULT_TOKEN +unset VAULT_ROLE_ID VAULT_SECRET_ID + +SONAR_TOKEN=$(vault read -field=token secret/logstash-ci/sonar-creds) +unset VAULT_TOKEN +DOCKER_ENV_OPTS="-e SONAR_TOKEN=${SONAR_TOKEN} -e SOURCE_BRANCH=$ghprbSourceBranch -e TARGET_BRANCH=$ghprbTargetBranch -e PULL_ID=$ghprbPullId -e COMMIT_SHA=$branch_specifier" \ + ci/docker_run.sh logstash-unit-tests ci/unit_tests.sh $@ diff --git a/ci/logstash_releases.json b/ci/logstash_releases.json index 449859c743e..2914d121167 100644 --- a/ci/logstash_releases.json +++ b/ci/logstash_releases.json @@ -2,11 +2,11 @@ "releases": { "5.x": "5.6.16", "6.x": "6.8.23", - "7.x": "7.17.12", - "8.x": "8.10.0" + "7.x": "7.17.13", + "8.x": "8.10.1" }, "snapshots": { - "7.x": "7.17.13-SNAPSHOT", + "7.x": "7.17.14-SNAPSHOT", "8.x": "8.11.0-SNAPSHOT" } } diff --git a/ci/unit_tests.sh b/ci/unit_tests.sh index d7a760d693e..07aa22280b2 100755 --- a/ci/unit_tests.sh +++ b/ci/unit_tests.sh @@ -19,7 +19,15 @@ SELECTED_TEST_SUITE=$1 if [[ $SELECTED_TEST_SUITE == $"java" ]]; then echo "Running Java Tests" - ./gradlew javaTests --console=plain --warning-mode all + ./gradlew javaTests jacocoTestReport sonar -Dsonar.token="${SONAR_TOKEN}" \ + -Dsonar.host.url=https://sonar.elastic.dev \ + -Dsonar.projectKey=elastic_logstash_AYm_nEbQaV3I-igkX1q9 \ + -Dsonar.projectName=logstash \ + -Dsonar.pullrequest.key=$PULL_ID \ + -Dsonar.pullrequest.branch=$SOURCE_BRANCH \ + -Dsonar.pullrequest.base=$TARGET_BRANCH \ + -Dsonar.scm.revision=$COMMIT_SHA \ + --console=plain --warning-mode all elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then echo "Running Ruby unit tests" ./gradlew rubyTests --console=plain --warning-mode all diff --git a/docs/static/releasenotes.asciidoc b/docs/static/releasenotes.asciidoc index 04c79b20fb8..642872bc29b 100644 --- a/docs/static/releasenotes.asciidoc +++ b/docs/static/releasenotes.asciidoc @@ -3,6 +3,7 @@ This section summarizes the changes in the following releases: +* <> * <> * <> * <> @@ -43,6 +44,11 @@ This section summarizes the changes in the following releases: * <> +[[logstash-8-10-1]] +=== Logstash 8.10.1 Release Notes + +No user-facing changes in Logstash core and plugins. + [[logstash-8-10-0]] === Logstash 8.10.0 Release Notes @@ -1628,4 +1634,4 @@ We have added another flag to the Benchmark CLI to allow passing a data file wit This feature allows users to run the Benchmark CLI in a custom test case with a custom config and a custom dataset. https://github.com/elastic/logstash/pull/12437[#12437] ==== Plugin releases -Plugins align with release 7.14.0 \ No newline at end of file +Plugins align with release 7.14.0 diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 586f1ca3d4d..a5b90f74a80 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -17,6 +17,39 @@ * under the License. */ + +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath "org.yaml:snakeyaml:${snakeYamlVersion}" + } +} + +plugins { + id "jacoco" + id "org.sonarqube" version "4.3.0.3225" +} + +apply plugin: 'jacoco' +apply plugin: "org.sonarqube" + +repositories { + mavenCentral() +} + +sonarqube { + properties { + property 'sonar.coverage.jacoco.xmlReportPaths', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml" + } +} + +jacoco { + toolVersion = "0.8.9" +} + + import org.yaml.snakeyaml.Yaml // fetch version from Logstash's main versions.yml file @@ -30,19 +63,6 @@ String jrubyVersion = versionMap['jruby']['version'] String log4jVersion = '2.17.1' -repositories { - mavenCentral() -} - -buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath "org.yaml:snakeyaml:${snakeYamlVersion}" - } -} - tasks.register("sourcesJar", Jar) { dependsOn classes from sourceSets.main.allSource @@ -102,8 +122,23 @@ tasks.register("javaTests", Test) { exclude '/org/logstash/plugins/CounterMetricImplTest.class' exclude '/org/logstash/plugins/factory/PluginFactoryExtTest.class' exclude '/org/logstash/execution/ObservedExecutionTest.class' + + jacoco { + enabled = true + destinationFile = layout.buildDirectory.file('jacoco/test.exec').get().asFile + classDumpDir = layout.buildDirectory.dir('jacoco/classpathdumps').get().asFile + } } +jacocoTestReport { + reports { + xml.enabled true + html.enabled true + } +} + +javaTests.finalizedBy(jacocoTestReport) + tasks.register("rubyTests", Test) { inputs.files fileTree("${projectDir}/lib") inputs.files fileTree("${projectDir}/spec") diff --git a/logstash-core/src/main/java/org/logstash/execution/PipelineReporterExt.java b/logstash-core/src/main/java/org/logstash/execution/PipelineReporterExt.java index 2cf4ac5590d..7c455253ec2 100644 --- a/logstash-core/src/main/java/org/logstash/execution/PipelineReporterExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/PipelineReporterExt.java @@ -37,6 +37,7 @@ import org.logstash.config.ir.compiler.AbstractOutputDelegatorExt; import java.util.Collection; +import java.util.Optional; /** * JRuby extension @@ -161,7 +162,7 @@ private RubyArray workerStates(final ThreadContext context, final RubyHash batch final RubyArray result = context.runtime.newArray(); ((Iterable) pipeline.callMethod(context, "worker_threads")) .forEach(thread -> { - final long nativeThreadId = ((RubyThread) thread).getNativeThread().getId(); + final RubyHash hash = RubyHash.newHash(context.runtime); IRubyObject status = thread.callMethod(context, "status"); if (status.isNil()) { @@ -170,8 +171,15 @@ private RubyArray workerStates(final ThreadContext context, final RubyHash batch hash.op_aset(context, STATUS_KEY, status); hash.op_aset(context, ALIVE_KEY, thread.callMethod(context, "alive?")); hash.op_aset(context, INDEX_KEY, context.runtime.newFixnum(result.size())); - final IRubyObject batch = batchMap.op_aref(context, context.runtime.newFixnum(nativeThreadId)); - hash.op_aset(context, INFLIGHT_COUNT_KEY, extractBatchSize(context, batch)); + + IRubyObject batchSize = Optional.of((RubyThread) thread) + .map(RubyThread::getNativeThread) + .map(Thread::getId) + .map(id -> batchMap.op_aref(context, context.runtime.newFixnum(id))) + .map(batch -> extractBatchSize(context, batch)) + .orElse(context.runtime.newFixnum(0L)); + + hash.op_aset(context, INFLIGHT_COUNT_KEY, batchSize); result.add(hash); }); return result;