[buildkite] DRA staging + snapshot pipelines (#15344)

This commit is the step after #15337 and switches away from the generically named [logstash-dra-pipeline-ci](https://buildkite.com/elastic/logstash-dra-pipeline-ci) pipeline to two separate pipelines for `snapshot` and `staging`, reflecting the current structure in [Jenkins](https://logstash-ci.elastic.co/view/DRA/). We define a daily schedule for `snapshot`, but it's commented out for now, until we ramp down the Jenkins job. Finally, as we've created a dedicated Buildkite team for `logstash`, we place both pipelines under this team. Relates: - elastic/ingest-dev#1720 - #15337
elastic · Oct 2, 2023 · da51a6a · da51a6a
1 parent 05c2627
commit da51a6a
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 19 deletions.
diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml
@@ -1,6 +1,5 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
-## TODO rename this file to dra_pipeline_snapshot (and change the respective definition in .pipelines.yaml)
 steps:
   - input: "Build parameters"
     if: build.source != "schedule"
@@ -26,11 +25,9 @@ steps:
 
   - label: ":pipeline: Generate steps"
     command: |
-      set -eo pipefail
+      set -euo pipefail
 
-      export WORKFLOW_TYPE="snapshot"
+      echo "--- Building [${WORKFLOW_TYPE}] artifacts"
       python3 -m pip install pyyaml
-      echo "--- Generating dynamic pipeline steps:"
-      python3 .buildkite/scripts/dra/generatesteps.py
-      
+      echo "--- Building dynamic pipeline steps"
       python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload
diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py
@@ -19,7 +19,7 @@ def package_x86_step(branch, workflow_type):
   key: "logstash_build_packages_dra"
   agents:
     provider: gcp
-    imageProject: elastic-images-qa
+    imageProject: elastic-images-prod
     image: family/platform-ingest-logstash-ubuntu-2204
     machineType: "n2-standard-16"
     diskSizeGb: 200
@@ -38,7 +38,7 @@ def package_x86_docker_step(branch, workflow_type):
   key: "logstash_build_x86_64_docker_dra"
   agents:
     provider: gcp
-    imageProject: elastic-images-qa
+    imageProject: elastic-images-prod
     image: family/platform-ingest-logstash-ubuntu-2204
     machineType: "n2-standard-16"
     diskSizeGb: 200
@@ -78,7 +78,7 @@ def publish_dra_step(branch, workflow_type, depends_on):
   depends_on: "{depends_on}"
   agents:
     provider: gcp
-    imageProject: elastic-images-qa
+    imageProject: elastic-images-prod
     image: family/platform-ingest-logstash-ubuntu-2204
     machineType: "n2-standard-16"
     diskSizeGb: 200

diff --git a/catalog-info.yaml b/catalog-info.yaml
@@ -26,8 +26,8 @@ spec:
   dependsOn:
     - resource:buildkite-logstash-serverless-integration-testing
     - resource:logstash-snyk-report
-    - resource:logstash-dra-pipeline
-
+    - logstash-dra-snapshot-pipeline
+    - logstash-dra-staging-pipeline
 
 # ***********************************
 # Declare serverless IT pipeline
@@ -110,39 +110,105 @@ spec:
           message: "Run the Logstash Snyk report every day."
 
 # ***********************************
-# Declare DRA pipelines
+# SECTION START: DRA pipelines
 # ***********************************
+
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
 kind: Resource
 metadata:
-  name: logstash-dra-pipeline
-  description: 'The logstash DRA pipeline.'
+  name: logstash-dra-snapshot-pipeline
+  description: Logstash Snapshot DRA pipeline
   links:
-    - title: Pipeline
-      url: https://buildkite.com/elastic/logstash-dra-pipeline-ci
+    - title: Logstash Snapshot DRA pipeline
+      url: https://buildkite.com/elastic/logstash-dra-snapshot-pipeline
 spec:
   type: buildkite-pipeline
-  owner: group:ingest-fp
+  owner: group:logstash
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: logstash-dra-snapshot-pipeline
+      description: ':logstash: The DRA snapshot :pipeline:'
+    spec:
+      repository: elastic/logstash
+      pipeline_file: ".buildkite/dra_pipeline.yml"
+      # TODO: uncomment out the schedule after testing + disabling Jenkins Job
+      # schedules:
+      #   Daily 7_17:
+      #     branch: '7.17'
+      #     cronline: 30 01 * * *
+      #     message: Daily SNAPSHOT build for 7.17 
+      #   Daily 8_10:
+      #     branch: '8.10'
+      #     cronline: 30 01 * * *
+      #     message: Daily SNAPSHOT build for 8.10 
+      #   Daily main:
+      #     branch: main
+      #     cronline: 30 01 * * *
+      #     message: Daily SNAPSHOT build for main
+      skip_intermediate_builds: true
+      provider_settings:
+        trigger_mode: none
+      env:
+        WORKFLOW_TYPE: 'snapshot'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' # don't alert during development
+        SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
+        SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
+      teams:
+        ingest-fp:
+          access_level: MANAGE_BUILD_AND_READ
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY
+
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: logstash-dra-staging-pipeline
+  description: 'The logstash DRA staging pipeline.'
+  links:
+    - title: Logstash Staging DRA pipeline
+      url: https://buildkite.com/elastic/logstash-dra-staging-pipeline
+spec:
+  type: buildkite-pipeline
+  owner: group:logstash
   system: buildkite
   implementation:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      name: logstash-dra-pipeline-ci
-      description: ':logstash: The DRA :pipeline:'
+      name: logstash-dra-staging-pipeline
+      description: ':logstash: The DRA staging :pipeline:'
     spec:
       repository: elastic/logstash
       pipeline_file: ".buildkite/dra_pipeline.yml"
+      skip_intermediate_builds: true
       provider_settings:
         trigger_mode: none
       env:
+        WORKFLOW_TYPE: 'staging'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false' # don't alert during development
         SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
         SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
       teams:
         ingest-fp:
           access_level: MANAGE_BUILD_AND_READ
+        logstash:
+          access_level: MANAGE_BUILD_AND_READ
+        ingest-eng-prod:
+          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
+
+# ***********************************
+# SECTION END: DRA pipelines
+# ***********************************