From da51a6ac53296b0c200db22c84d046f002ee5570 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 2 Oct 2023 12:10:59 +0300 Subject: [PATCH] [buildkite] DRA staging + snapshot pipelines (#15344) This commit is the step after #15337 and switches away from the generically named [logstash-dra-pipeline-ci](https://buildkite.com/elastic/logstash-dra-pipeline-ci) pipeline to two separate pipelines for `snapshot` and `staging`, reflecting the current structure in [Jenkins](https://logstash-ci.elastic.co/view/DRA/). We define a daily schedule for `snapshot`, but it's commented out for now, until we ramp down the Jenkins job. Finally, as we've created a dedicated Buildkite team for `logstash`, we place both pipelines under this team. Relates: - https://github.com/elastic/ingest-dev/issues/1720 - https://github.com/elastic/logstash/pull/15337 --- .buildkite/dra_pipeline.yml | 9 +-- .buildkite/scripts/dra/generatesteps.py | 6 +- catalog-info.yaml | 86 ++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 19 deletions(-) diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml index a7821022d70..f0bb01ea94e 100644 --- a/.buildkite/dra_pipeline.yml +++ b/.buildkite/dra_pipeline.yml @@ -1,6 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json -## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml) steps: - input: "Build parameters" if: build.source != "schedule" @@ -26,11 +25,9 @@ steps: - label: ":pipeline: Generate steps" command: | - set -eo pipefail + set -euo pipefail - export WORKFLOW_TYPE="snapshot" + echo "--- Building [${WORKFLOW_TYPE}] artifacts" python3 -m pip install pyyaml - echo "--- Generating dynamic pipeline steps:" - python3 .buildkite/scripts/dra/generatesteps.py - + echo "--- Building dynamic pipeline steps" python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py index 64ce387d4d7..1cc05cffe0a 100644 --- a/.buildkite/scripts/dra/generatesteps.py +++ b/.buildkite/scripts/dra/generatesteps.py @@ -19,7 +19,7 @@ def package_x86_step(branch, workflow_type): key: "logstash_build_packages_dra" agents: provider: gcp - imageProject: elastic-images-qa + imageProject: elastic-images-prod image: family/platform-ingest-logstash-ubuntu-2204 machineType: "n2-standard-16" diskSizeGb: 200 @@ -38,7 +38,7 @@ def package_x86_docker_step(branch, workflow_type): key: "logstash_build_x86_64_docker_dra" agents: provider: gcp - imageProject: elastic-images-qa + imageProject: elastic-images-prod image: family/platform-ingest-logstash-ubuntu-2204 machineType: "n2-standard-16" diskSizeGb: 200 @@ -78,7 +78,7 @@ def publish_dra_step(branch, workflow_type, depends_on): depends_on: "{depends_on}" agents: provider: gcp - imageProject: elastic-images-qa + imageProject: elastic-images-prod image: family/platform-ingest-logstash-ubuntu-2204 machineType: "n2-standard-16" diskSizeGb: 200 diff --git a/catalog-info.yaml b/catalog-info.yaml index 452b60a3330..cb7e9d17fa9 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -26,8 +26,8 @@ spec: dependsOn: - resource:buildkite-logstash-serverless-integration-testing - resource:logstash-snyk-report - - resource:logstash-dra-pipeline - + - logstash-dra-snapshot-pipeline + - logstash-dra-staging-pipeline # *********************************** # Declare serverless IT pipeline @@ -110,39 +110,105 @@ spec: message: "Run the Logstash Snyk report every day." # *********************************** -# Declare DRA pipelines +# SECTION START: DRA pipelines # *********************************** + --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 kind: Resource metadata: - name: logstash-dra-pipeline - description: 'The logstash DRA pipeline.' + name: logstash-dra-snapshot-pipeline + description: Logstash Snapshot DRA pipeline links: - - title: Pipeline - url: https://buildkite.com/elastic/logstash-dra-pipeline-ci + - title: Logstash Snapshot DRA pipeline + url: https://buildkite.com/elastic/logstash-dra-snapshot-pipeline spec: type: buildkite-pipeline - owner: group:ingest-fp + owner: group:logstash + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: logstash-dra-snapshot-pipeline + description: ':logstash: The DRA snapshot :pipeline:' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/dra_pipeline.yml" + # TODO: uncomment out the schedule after testing + disabling Jenkins Job + # schedules: + # Daily 7_17: + # branch: '7.17' + # cronline: 30 01 * * * + # message: Daily SNAPSHOT build for 7.17 + # Daily 8_10: + # branch: '8.10' + # cronline: 30 01 * * * + # message: Daily SNAPSHOT build for 8.10 + # Daily main: + # branch: main + # cronline: 30 01 * * * + # message: Daily SNAPSHOT build for main + skip_intermediate_builds: true + provider_settings: + trigger_mode: none + env: + WORKFLOW_TYPE: 'snapshot' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' # don't alert during development + SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' + SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-dra-staging-pipeline + description: 'The logstash DRA staging pipeline.' + links: + - title: Logstash Staging DRA pipeline + url: https://buildkite.com/elastic/logstash-dra-staging-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash system: buildkite implementation: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - name: logstash-dra-pipeline-ci - description: ':logstash: The DRA :pipeline:' + name: logstash-dra-staging-pipeline + description: ':logstash: The DRA staging :pipeline:' spec: repository: elastic/logstash pipeline_file: ".buildkite/dra_pipeline.yml" + skip_intermediate_builds: true provider_settings: trigger_mode: none env: + WORKFLOW_TYPE: 'staging' ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' # don't alert during development SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ everyone: access_level: READ_ONLY + +# *********************************** +# SECTION END: DRA pipelines +# ***********************************