Run BK exhaustive pipeline when code is pushed (#15738)

This commit enables running the exhaustive tests Buildkite pipeline
(i.e. the equivalent to the `main` Jenkins tests) ; the trigger is
code events, i.e. direct pushes, merge commits and creation of new branches.

CI is skipped if changes are only related to files under `docs/`.

(cherry picked from commit c8726b7)

.buildkite/exhaustive_tests_pipeline.yml

@@ -2,9 +2,19 @@ steps:
   - label: "Exhaustive tests pipeline"
     command: |
       #!/usr/bin/env bash
+      echo "--- Check for docs changes"
+      set +e
+      .buildkite/scripts/common/check-files-changed.sh '^docs/.*'
+      if [[ $$? -eq 0 ]]; then
+        echo "^^^ +++"
+        echo "Skipping running pipeline as all changes are related to docs."
+        exit 0
+      else
+        echo "Changes are not exclusively related to docs, continuing."
+      fi
+
       set -eo pipefail
 
-      source .buildkite/scripts/common/container-agent.sh
       echo "--- Downloading prerequisites"
       python3 -m pip install ruamel.yaml
       curl -fsSL --retry-max-time 60 --retry 3 --retry-delay 5 -o /usr/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64

.buildkite/scripts/common/check-files-changed.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+# **********************************************************
+# Returns true if current checkout compared to parent commit
+# has changes ONLY matching the argument regexp
+#
+# Used primarily to skip running the exhaustive pipeline
+# when only docs changes have happened.
+# ********************************************************
+
+if [[ -z "$1" ]]; then
+  echo "Usage: $0 <regexp>"
+  exit 1
+fi
+
+previous_commit=$(git rev-parse HEAD^)
+changed_files=$(git diff --name-only $previous_commit)
+
+if [[ -n "$changed_files" ]] && [[ -z "$(echo "$changed_files" | grep -vE "$1")" ]]; then
+    echo "All files compared to the previous commit [$previous_commit] match the specified regex: [$1]"
+    echo "Files changed:"
+    git diff --name-only HEAD^
+  exit 0
+else
+  exit 1
+fi

catalog-info.yaml

@@ -475,11 +475,15 @@ spec:
       repository: elastic/logstash
       pipeline_file: ".buildkite/exhaustive_tests_pipeline.yml"
       provider_settings:
-        trigger_mode: none
-      cancel_intermediate_builds: true
-      skip_intermediate_builds: true
+        build_branches: true
+        build_pull_request_forks: false
+        build_pull_requests: false
+        build_tags: false
+        trigger_mode: code
+      cancel_intermediate_builds: false
+      skip_intermediate_builds: false
       env:
-        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'false'  # disable during development
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
         SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build'
         SLACK_NOTIFICATIONS_ON_SUCCESS: 'false'
       teams: