From e3b3d2a7ad5648e1c322fa8ff69d354db177c922 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Fri, 3 Nov 2023 12:37:49 +0200
Subject: [PATCH 01/17] [ci] JDK matrix Buildkite pipelines (part 1)

This commit is the first part of the migration of JDK matrix tests
from Jenkins to Buildkite. There will be two separate pipelines, for
Linux and Windows, and this commit only adds the Linux one.

Linux is currently limited to Ubuntu 22.04 and 20.04, but
additional operating systems will be added outside of the Logstash
repository seamlessly through additional VM images.

Steps are created dynamically and the underlying script is meant to be
common for Linux and Windows. Windows is currently a stub and
will be added in a follow up PR.

Relates:

- https://github.com/elastic/ingest-dev/issues/1725
- https://github.com/elastic/ci-agent-images/pull/424
---
 .buildkite/jdk_matrix_pipeline.yml            |  88 +++++++-
 .../jdk-matrix-tests/generate-steps.py        | 199 ++++++++++++++++++
 2 files changed, 285 insertions(+), 2 deletions(-)
 create mode 100644 .buildkite/scripts/jdk-matrix-tests/generate-steps.py

diff --git a/.buildkite/jdk_matrix_pipeline.yml b/.buildkite/jdk_matrix_pipeline.yml
index 307057543ff..3c2905176e5 100644
--- a/.buildkite/jdk_matrix_pipeline.yml
+++ b/.buildkite/jdk_matrix_pipeline.yml
@@ -1,3 +1,87 @@
 steps:
-  - label: "Test JDK matrix pipeline"
-    command: "echo 'Hello world'"
+  - input: "Test Parameters"
+    if: build.source != "schedule"
+    fields:
+      - select: "Operating System"
+        key: "matrix-os"
+        hint: "The operating system variant(s) to run on:"
+        required: true
+        multiple: true
+        default: "ubuntu-2204"
+        options:
+          - label: "Ubuntu 22.04"
+            value: "ubuntu-2204"
+          - label: "Ubuntu 20.04"
+            value: "ubuntu-2004"
+          - label: "Ubuntu 18.04"
+            value: "ubuntu-1804"
+          - label: "Debian 11"
+            value: "debian-11"
+          - label: "Debian 10"
+            value: "debian-10"
+          - label: "RHEL 9"
+            value: "rhel-9"
+          - label: "RHEL 8"
+            value: "rhel-8"
+          - label: "CentOS 8"
+            value: "centos-8"
+          - label: "CentOS 7"
+            value: "centos-7"
+          - label: "Oracle Linux 8"
+            value: "oraclelinux-8"
+          - label: "Oracle Linux 7"
+            value: "oraclelinux-7"
+          - label: "Rocky Linux 8"
+            value: "rocky-8"
+          - label: "Amazon Linux"
+            value: "amazonlinux-2023"
+          - label: "OpenSUSE Leap 15"
+            value: "opensuse-leap-15"
+
+      - select: "Java"
+        key: "matrix-jdk"
+        hint: "The JDK to test with:"
+        required: true
+        multiple: true
+        default: "adoptiumjdk_17"
+        options:
+          - label: "Adoptium JDK 17 (Eclipse Temurin)"
+            value: "adoptiumjdk_17"
+          - label: "Adopt OpenJDK 11"
+            value: "adoptopenjdk_11"
+          - label: "OpenJDK 17"
+            value: "openjdk_17"
+          - label: "OpenJDK 11"
+            value: "openjdk_11"
+          - label: "Zulu 17"
+            value: "zulu_17"
+          - label: "Zulu 11"
+            value: "zulu_11"
+
+  - wait: ~
+    if: build.source != "schedule"
+
+  - command: |
+      set -euo pipefail
+
+      echo "--- Downloading prerequisites"
+      python3 -m pip install ruamel.yaml
+
+      echo "--- Printing generated dynamic steps"
+      export MATRIX_OSES="$(buildkite-agent meta-data get matrix-os)"
+      export MATRIX_JDKS="$(buildkite-agent meta-data get matrix-jdk)"
+      set +eo pipefail
+      python3 .buildkite/scripts/jdk-matrix-tests/generate-steps.py >pipeline_steps.yml
+      if [[ $$? -ne 0 ]]; then
+        echo "^^^ +++"
+        echo "There was a problem rendering the pipeline steps."
+        cat pipeline_steps.yml
+        echo "Exiting now."
+        exit 1
+      else
+        set -eo pipefail
+        cat pipeline_steps.yml
+      fi
+
+      echo "--- Uploading steps to buildkite"
+      cat pipeline_steps.yml | buildkite-agent pipeline upload
diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
new file mode 100644
index 00000000000..1ed1c6b8c6a
--- /dev/null
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -0,0 +1,199 @@
+from dataclasses import dataclass
+import os
+import sys
+import typing
+
+from ruamel.yaml import YAML
+from ruamel.yaml.scalarstring import LiteralScalarString
+
+
+def get_bk_metadata(key: str) -> typing.List[str]:
+    try:
+      return os.environ[key].split()
+    except KeyError:
+        print(f"Missing environment variable [{key}]. This should be set before calling this script using buildkite-agent meta-data get. Exiting.")
+        exit(1)
+
+def bk_annotate(job_name_human: str, job_name_slug: str, os: str, jdk: str, status: str) -> str:
+  return f"""buildkite-agent annotate "{status} **{job_name_human}** / **{os}** / **{jdk}**" --context={job_name_slug}-{os}-{jdk}"""
+
+
+@dataclass
+class BuildkiteEmojis:
+  running: str = ":bk-status-running:"
+  success: str = ":bk-status-passed:"
+  failed: str = ":bk-status-failed:"
+
+
+class WindowsJobs:
+    def __init__(self, os: str, jdk: str):
+      self.os = os
+      self.jdk = jdk
+
+    def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
+        return [
+          self.unit_tests,
+        ]
+
+    def unit_tests(self) -> typing.Tuple[str, str]:
+        job_name_human = "Java Unit Test"
+        job_name_slug = "java-unit-test"
+        test_command = "# TODO"
+
+        return job_name_human, test_command
+      
+
+class LinuxJobs:
+    def __init__(self, os: str, jdk: str):
+      self.os = os
+      self.jdk = jdk
+
+    def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
+        return [
+            self.java_unit_test,
+            self.ruby_unit_test,
+            ## temporarily disabled due to https://github.com/elastic/logstash/issues/15529
+            # self.integration_tests_part_1,
+            self.integration_tests_part_2,
+            ## temporarily disabled due to https://github.com/elastic/logstash/issues/15529
+            # self.pq_integration_tests_part_1,
+            self.pq_integration_tests_part_2,
+            self.x_pack_unit_tests,
+            self.x_pack_integration,
+        ]
+
+    def prepare_shell(self) -> str:
+        jdk_dir = f"/opt/buildkite-agent/.java/{self.jdk}"
+        return f"""#!/usr/bin/env bash
+set -euo pipefail
+
+# unset generic JAVA_HOME
+unset JAVA_HOME
+
+# LS env vars for JDK matrix tests
+export BUILD_JAVA_HOME={jdk_dir}
+export RUNTIME_JAVA_HOME={jdk_dir}
+export LS_JAVA_HOME={jdk_dir}
+
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
+eval "$(rbenv init -)"
+"""
+
+    def emit_command(self, job_name_human, job_name_slug, test_command: str) -> str:
+      return LiteralScalarString(f"""
+{self.prepare_shell()}
+{bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.running)}
+# temporarily disable immediate failure on errors, so that we can update the BK annotation
+set +eo pipefail
+{test_command}
+if [[ $$? -ne 0 ]]; then
+  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.failed)}
+  exit 1
+else
+  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.success)}
+fi
+      """)
+
+    def java_unit_test(self) -> typing.Tuple[str, str]:
+        job_name_human = "Java Unit Test"
+        job_name_slug = "java-unit-test"
+        test_command = '''
+export ENABLE_SONARQUBE="false"
+ci/unit_tests.sh java
+        '''
+
+        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+
+    def ruby_unit_test(self) -> typing.Tuple[str, str]:
+        job_name_human = "Ruby Unit Test"
+        job_name_slug = "ruby-unit-test"
+        test_command = """
+ci/unit_tests.sh ruby
+        """
+
+        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+
+    def integration_tests_part_1(self) -> typing.Tuple[str, str]:
+        return self.integration_tests(part=1)
+
+    def integration_tests_part_2(self) -> typing.Tuple[str, str]:
+        return self.integration_tests(part=2)
+
+    def integration_tests(self, part: int) -> typing.Tuple[str, str]:
+        job_name_human = f"Integration Tests - {part}"
+        job_name_slug = f"integration-tests-pt-{part}"
+        test_command = f"""
+ci/integration_tests.sh split {part-1}
+        """
+
+        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+
+    def pq_integration_tests_part_1(self) -> typing.Tuple[str, str]:
+        return self.pq_integration_tests(part=1)
+
+    def pq_integration_tests_part_2(self) -> typing.Tuple[str, str]:
+        return self.pq_integration_tests(part=2)
+
+    def pq_integration_tests(self, part: int) -> typing.Tuple[str, str]:
+        job_name_human = f"IT Persistent Queues - {part}"
+        job_name_slug = f"it-persistent-queues-pt-{part}"
+        test_command = f"""
+export FEATURE_FLAG=persistent_queues
+ci/integration_tests.sh split {part-1}
+        """
+
+        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+
+    def x_pack_unit_tests(self) -> typing.Tuple[str, str]:
+        job_name_human = "x-pack unit tests"
+        job_name_slug = "x-pack-unit-tests"
+        test_command = """
+x-pack/ci/unit_tests.sh
+        """
+
+        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+
+    def x_pack_integration(self) -> typing.Tuple[str, str]:
+        job_name_human = "x-pack integration"
+        job_name_slug = "x-pack-integration"
+        test_command = """
+x-pack/ci/integration_tests.sh
+        """
+
+        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+
+
+if __name__ == "__main__":
+    matrix_oses = get_bk_metadata(key="MATRIX_OSES")
+    matrix_jdkes = get_bk_metadata(key="MATRIX_JDKS")
+
+    pipeline_name = os.environ.get("BUILDKITE_PIPELINE_NAME", "").lower()
+
+    structure = {"steps": []}
+
+
+    for matrix_os in matrix_oses:
+        for matrix_jdk in matrix_jdkes:
+          if "windows" in pipeline_name:
+            jobs = WindowsJobs(os=matrix_os, jdk=matrix_jdk)
+          else:
+            jobs = LinuxJobs(os=matrix_os, jdk=matrix_jdk)
+
+          for job in jobs.all_jobs():
+            job_name_human, shell_command = job()
+            step = {
+              "label": f"{matrix_os} / {matrix_jdk} / {job_name_human}",
+              "agents": {
+                  "provider": "gcp",
+                  "imageProject": "elastic-images-qa",
+                  "image": f"family/platform-ingest-logstash-multi-jdk-{matrix_os}",
+                  "machineType": "n2-standard-4",
+                  "diskSizeGb": 200,
+                  "diskType": "pd-ssd",
+                },
+                "command": shell_command,
+            }
+
+            structure["steps"].append(step)
+
+    YAML().dump(structure, sys.stdout)

From 814c1ae3ad59e0c0f5cfea627648f1a2fce3883e Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Tue, 7 Nov 2023 16:35:32 +0200
Subject: [PATCH 02/17] Resolve conflicts

---
 .buildkite/linux_jdk_matrix_pipeline.yml | 88 +++++++++++++++++++++++-
 1 file changed, 86 insertions(+), 2 deletions(-)

diff --git a/.buildkite/linux_jdk_matrix_pipeline.yml b/.buildkite/linux_jdk_matrix_pipeline.yml
index d7ada8ec87c..3c2905176e5 100644
--- a/.buildkite/linux_jdk_matrix_pipeline.yml
+++ b/.buildkite/linux_jdk_matrix_pipeline.yml
@@ -1,3 +1,87 @@
 steps:
-  - label: "Test Linux JDK matrix pipeline"
-    command: "echo 'Hello world'"
+  - input: "Test Parameters"
+    if: build.source != "schedule"
+    fields:
+      - select: "Operating System"
+        key: "matrix-os"
+        hint: "The operating system variant(s) to run on:"
+        required: true
+        multiple: true
+        default: "ubuntu-2204"
+        options:
+          - label: "Ubuntu 22.04"
+            value: "ubuntu-2204"
+          - label: "Ubuntu 20.04"
+            value: "ubuntu-2004"
+          - label: "Ubuntu 18.04"
+            value: "ubuntu-1804"
+          - label: "Debian 11"
+            value: "debian-11"
+          - label: "Debian 10"
+            value: "debian-10"
+          - label: "RHEL 9"
+            value: "rhel-9"
+          - label: "RHEL 8"
+            value: "rhel-8"
+          - label: "CentOS 8"
+            value: "centos-8"
+          - label: "CentOS 7"
+            value: "centos-7"
+          - label: "Oracle Linux 8"
+            value: "oraclelinux-8"
+          - label: "Oracle Linux 7"
+            value: "oraclelinux-7"
+          - label: "Rocky Linux 8"
+            value: "rocky-8"
+          - label: "Amazon Linux"
+            value: "amazonlinux-2023"
+          - label: "OpenSUSE Leap 15"
+            value: "opensuse-leap-15"
+
+      - select: "Java"
+        key: "matrix-jdk"
+        hint: "The JDK to test with:"
+        required: true
+        multiple: true
+        default: "adoptiumjdk_17"
+        options:
+          - label: "Adoptium JDK 17 (Eclipse Temurin)"
+            value: "adoptiumjdk_17"
+          - label: "Adopt OpenJDK 11"
+            value: "adoptopenjdk_11"
+          - label: "OpenJDK 17"
+            value: "openjdk_17"
+          - label: "OpenJDK 11"
+            value: "openjdk_11"
+          - label: "Zulu 17"
+            value: "zulu_17"
+          - label: "Zulu 11"
+            value: "zulu_11"
+
+  - wait: ~
+    if: build.source != "schedule"
+
+  - command: |
+      set -euo pipefail
+
+      echo "--- Downloading prerequisites"
+      python3 -m pip install ruamel.yaml
+
+      echo "--- Printing generated dynamic steps"
+      export MATRIX_OSES="$(buildkite-agent meta-data get matrix-os)"
+      export MATRIX_JDKS="$(buildkite-agent meta-data get matrix-jdk)"
+      set +eo pipefail
+      python3 .buildkite/scripts/jdk-matrix-tests/generate-steps.py >pipeline_steps.yml
+      if [[ $$? -ne 0 ]]; then
+        echo "^^^ +++"
+        echo "There was a problem rendering the pipeline steps."
+        cat pipeline_steps.yml
+        echo "Exiting now."
+        exit 1
+      else
+        set -eo pipefail
+        cat pipeline_steps.yml
+      fi
+
+      echo "--- Uploading steps to buildkite"
+      cat pipeline_steps.yml | buildkite-agent pipeline upload

From c7849fec13855b8423579247d10432c4e4cd86c1 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Tue, 7 Nov 2023 16:39:49 +0200
Subject: [PATCH 03/17] Remove old pipeline after fixing conflicts

---
 .buildkite/jdk_matrix_pipeline.yml | 87 ------------------------------
 1 file changed, 87 deletions(-)
 delete mode 100644 .buildkite/jdk_matrix_pipeline.yml

diff --git a/.buildkite/jdk_matrix_pipeline.yml b/.buildkite/jdk_matrix_pipeline.yml
deleted file mode 100644
index 3c2905176e5..00000000000
--- a/.buildkite/jdk_matrix_pipeline.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-steps:
-  - input: "Test Parameters"
-    if: build.source != "schedule"
-    fields:
-      - select: "Operating System"
-        key: "matrix-os"
-        hint: "The operating system variant(s) to run on:"
-        required: true
-        multiple: true
-        default: "ubuntu-2204"
-        options:
-          - label: "Ubuntu 22.04"
-            value: "ubuntu-2204"
-          - label: "Ubuntu 20.04"
-            value: "ubuntu-2004"
-          - label: "Ubuntu 18.04"
-            value: "ubuntu-1804"
-          - label: "Debian 11"
-            value: "debian-11"
-          - label: "Debian 10"
-            value: "debian-10"
-          - label: "RHEL 9"
-            value: "rhel-9"
-          - label: "RHEL 8"
-            value: "rhel-8"
-          - label: "CentOS 8"
-            value: "centos-8"
-          - label: "CentOS 7"
-            value: "centos-7"
-          - label: "Oracle Linux 8"
-            value: "oraclelinux-8"
-          - label: "Oracle Linux 7"
-            value: "oraclelinux-7"
-          - label: "Rocky Linux 8"
-            value: "rocky-8"
-          - label: "Amazon Linux"
-            value: "amazonlinux-2023"
-          - label: "OpenSUSE Leap 15"
-            value: "opensuse-leap-15"
-
-      - select: "Java"
-        key: "matrix-jdk"
-        hint: "The JDK to test with:"
-        required: true
-        multiple: true
-        default: "adoptiumjdk_17"
-        options:
-          - label: "Adoptium JDK 17 (Eclipse Temurin)"
-            value: "adoptiumjdk_17"
-          - label: "Adopt OpenJDK 11"
-            value: "adoptopenjdk_11"
-          - label: "OpenJDK 17"
-            value: "openjdk_17"
-          - label: "OpenJDK 11"
-            value: "openjdk_11"
-          - label: "Zulu 17"
-            value: "zulu_17"
-          - label: "Zulu 11"
-            value: "zulu_11"
-
-  - wait: ~
-    if: build.source != "schedule"
-
-  - command: |
-      set -euo pipefail
-
-      echo "--- Downloading prerequisites"
-      python3 -m pip install ruamel.yaml
-
-      echo "--- Printing generated dynamic steps"
-      export MATRIX_OSES="$(buildkite-agent meta-data get matrix-os)"
-      export MATRIX_JDKS="$(buildkite-agent meta-data get matrix-jdk)"
-      set +eo pipefail
-      python3 .buildkite/scripts/jdk-matrix-tests/generate-steps.py >pipeline_steps.yml
-      if [[ $$? -ne 0 ]]; then
-        echo "^^^ +++"
-        echo "There was a problem rendering the pipeline steps."
-        cat pipeline_steps.yml
-        echo "Exiting now."
-        exit 1
-      else
-        set -eo pipefail
-        cat pipeline_steps.yml
-      fi
-
-      echo "--- Uploading steps to buildkite"
-      cat pipeline_steps.yml | buildkite-agent pipeline upload

From 2b7a455eca695164112709da7be0d0378634bfc5 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 13:11:13 +0200
Subject: [PATCH 04/17] Switch to groups

---
 .../jdk-matrix-tests/generate-steps.py        | 58 ++++++++++++++-----
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 1ed1c6b8c6a..0ab2a400a04 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -7,6 +7,22 @@
 from ruamel.yaml.scalarstring import LiteralScalarString
 
 
+@dataclass
+class BuildkiteEmojis:
+  running: str = ":bk-status-running:"
+  success: str = ":bk-status-passed:"
+  failed: str = ":bk-status-failed:"
+
+def slugify_bk_key(key: str) -> str:
+    """
+    Convert and return key to an acceptable format for Buildkite's key: field
+    Only alphanumerics, dashes and underscores are allowed.
+    """
+
+    mapping_table = str.maketrans({'.': '_', ' ': '_', '/': '_'})
+
+    return key.translate(mapping_table)
+
 def get_bk_metadata(key: str) -> typing.List[str]:
     try:
       return os.environ[key].split()
@@ -14,21 +30,19 @@ def get_bk_metadata(key: str) -> typing.List[str]:
         print(f"Missing environment variable [{key}]. This should be set before calling this script using buildkite-agent meta-data get. Exiting.")
         exit(1)
 
-def bk_annotate(job_name_human: str, job_name_slug: str, os: str, jdk: str, status: str) -> str:
-  return f"""buildkite-agent annotate "{status} **{job_name_human}** / **{os}** / **{jdk}**" --context={job_name_slug}-{os}-{jdk}"""
-
+def bk_annotate(job_name_human: str, job_name_slug: str, os: str, jdk: str, status: str, context: str, mode: str) -> str:
+  cmd = f"""buildkite-agent annotate "{status} **{job_name_human}** / **{os}** / **{jdk}**" --context={context}"""
+  if mode:
+     cmd += f" --{mode}"
 
-@dataclass
-class BuildkiteEmojis:
-  running: str = ":bk-status-running:"
-  success: str = ":bk-status-passed:"
-  failed: str = ":bk-status-failed:"
+  return cmd
 
 
 class WindowsJobs:
-    def __init__(self, os: str, jdk: str):
+    def __init__(self, os: str, jdk: str, group_key: str):
       self.os = os
       self.jdk = jdk
+      self.group_key: str
 
     def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
         return [
@@ -44,9 +58,10 @@ def unit_tests(self) -> typing.Tuple[str, str]:
       
 
 class LinuxJobs:
-    def __init__(self, os: str, jdk: str):
+    def __init__(self, os: str, jdk: str, group_key: str):
       self.os = os
       self.jdk = jdk
+      self.group_key = group_key
 
     def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
         return [
@@ -82,15 +97,14 @@ def prepare_shell(self) -> str:
     def emit_command(self, job_name_human, job_name_slug, test_command: str) -> str:
       return LiteralScalarString(f"""
 {self.prepare_shell()}
-{bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.running)}
 # temporarily disable immediate failure on errors, so that we can update the BK annotation
 set +eo pipefail
 {test_command}
 if [[ $$? -ne 0 ]]; then
-  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.failed)}
+  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.failed, context=self.group_key, mode="append")}
   exit 1
 else
-  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.success)}
+  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.success, context=self.group_key, mode="append")}
 fi
       """)
 
@@ -174,13 +188,18 @@ def x_pack_integration(self) -> typing.Tuple[str, str]:
 
     for matrix_os in matrix_oses:
         for matrix_jdk in matrix_jdkes:
+          group_name = f"{matrix_os}/{matrix_jdk}"
+          group_key = slugify_bk_key(group_name)
+
           if "windows" in pipeline_name:
-            jobs = WindowsJobs(os=matrix_os, jdk=matrix_jdk)
+            jobs = WindowsJobs(os=matrix_os, jdk=matrix_jdk, group_key=group_key)
           else:
-            jobs = LinuxJobs(os=matrix_os, jdk=matrix_jdk)
+            jobs = LinuxJobs(os=matrix_os, jdk=matrix_jdk, group_key=group_key)
 
+          group_steps = []
           for job in jobs.all_jobs():
             job_name_human, shell_command = job()
+
             step = {
               "label": f"{matrix_os} / {matrix_jdk} / {job_name_human}",
               "agents": {
@@ -194,6 +213,13 @@ def x_pack_integration(self) -> typing.Tuple[str, str]:
                 "command": shell_command,
             }
 
-            structure["steps"].append(step)
+            group_steps.append(step)
+
+
+          structure["steps"].append({
+            "group": group_name,
+            "key": slugify_bk_key(group_name),
+            "steps": group_steps})
+
 
     YAML().dump(structure, sys.stdout)

From 34b19113163cf7f1fa5bd7fac8780faae2dc5929 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 13:14:26 +0200
Subject: [PATCH 05/17] re-enable integration tests pt 1

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 0ab2a400a04..7bc672f3420 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -67,11 +67,9 @@ def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
         return [
             self.java_unit_test,
             self.ruby_unit_test,
-            ## temporarily disabled due to https://github.com/elastic/logstash/issues/15529
-            # self.integration_tests_part_1,
+            self.integration_tests_part_1,
             self.integration_tests_part_2,
-            ## temporarily disabled due to https://github.com/elastic/logstash/issues/15529
-            # self.pq_integration_tests_part_1,
+            self.pq_integration_tests_part_1,
             self.pq_integration_tests_part_2,
             self.x_pack_unit_tests,
             self.x_pack_integration,

From 717e7816a64d57bd2edcb1322ff7e55b79ecc8c3 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 16:51:35 +0200
Subject: [PATCH 06/17] Improved annotations

---
 .../jdk-matrix-tests/generate-steps.py        | 210 ++++++++++++------
 1 file changed, 146 insertions(+), 64 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 7bc672f3420..c378e069213 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -1,3 +1,4 @@
+import abc
 from dataclasses import dataclass
 import os
 import sys
@@ -7,11 +8,19 @@
 from ruamel.yaml.scalarstring import LiteralScalarString
 
 
+@dataclass
+class JobRetValues:
+    step_label: str
+    command: str
+    step_key: str
+    depends: str
+    default_agent: bool = False
+
 @dataclass
 class BuildkiteEmojis:
-  running: str = ":bk-status-running:"
-  success: str = ":bk-status-passed:"
-  failed: str = ":bk-status-failed:"
+    running: str = ":bk-status-running:"
+    success: str = ":bk-status-passed:"
+    failed: str = ":bk-status-failed:"
 
 def slugify_bk_key(key: str) -> str:
     """
@@ -30,41 +39,73 @@ def get_bk_metadata(key: str) -> typing.List[str]:
         print(f"Missing environment variable [{key}]. This should be set before calling this script using buildkite-agent meta-data get. Exiting.")
         exit(1)
 
-def bk_annotate(job_name_human: str, job_name_slug: str, os: str, jdk: str, status: str, context: str, mode: str) -> str:
-  cmd = f"""buildkite-agent annotate "{status} **{job_name_human}** / **{os}** / **{jdk}**" --context={context}"""
-  if mode:
-     cmd += f" --{mode}"
+def bk_annotate(body: str, context: str, mode = "") -> str:
+    cmd = f"""buildkite-agent annotate "{body}" --context={context}"""
+    if mode:
+        cmd += f" --{mode}"
 
-  return cmd
+    return cmd
 
 
-class WindowsJobs:
+class Jobs(abc.ABC):
     def __init__(self, os: str, jdk: str, group_key: str):
-      self.os = os
-      self.jdk = jdk
-      self.group_key: str
+        self.os = os
+        self.jdk = jdk
+        self.group_key = group_key
+        self.init_annotation_key = "initialize-annotation"
+
+    def init_annotation(self) -> JobRetValues:
+        """
+        Command for creating the header of a new annotation for a group step
+        """
+
+        body = f'''## Group {self.os} / {self.jdk}
+
+        | **Status** | **Test** |
+        '''
+
+        return JobRetValues(
+            step_label="Initialize annotation",
+            command=LiteralScalarString(bk_annotate(body=body, context=self.group_key)),
+            step_key=self.init_annotation_key,
+            depends="",
+            default_agent=True,
+        )
 
+    @abc.abstractmethod
     def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
+        pass
+
+
+class WindowsJobs(Jobs):
+    def __init__(self, os: str, jdk: str, group_key: str):
+      super().__init__(os=os, jdk=jdk, group_key=group_key)
+
+    def all_jobs(self):# -> list[Callable[[], JobRetValues]]:
         return [
           self.unit_tests,
         ]
 
-    def unit_tests(self) -> typing.Tuple[str, str]:
-        job_name_human = "Java Unit Test"
-        job_name_slug = "java-unit-test"
+    def unit_tests(self) -> JobRetValues:
+        step_name_human = "Java Unit Test"
         test_command = "# TODO"
 
-        return job_name_human, test_command
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key="java-unit-test",
+            depends="",
+        )
+        return step_name_human, test_command
       
 
-class LinuxJobs:
+class LinuxJobs(Jobs):
     def __init__(self, os: str, jdk: str, group_key: str):
-      self.os = os
-      self.jdk = jdk
-      self.group_key = group_key
+      super().__init__(os=os, jdk=jdk, group_key=group_key)
 
-    def all_jobs(self) -> list[typing.Callable[[], typing.Tuple[str, str]]]:
+    def all_jobs(self):# -> list[Callable[[], JobRetValues]]:
         return [
+            self.init_annotation,
             self.java_unit_test,
             self.ruby_unit_test,
             self.integration_tests_part_1,
@@ -92,87 +133,122 @@ def prepare_shell(self) -> str:
 eval "$(rbenv init -)"
 """
 
-    def emit_command(self, job_name_human, job_name_slug, test_command: str) -> str:
-      return LiteralScalarString(f"""
+    def failed_step_annotation(self, step_name_human) -> str:
+        return bk_annotate(body=f"| {BuildkiteEmojis.failed} | {step_name_human} |\n", context=self.group_key, mode="append")
+
+    def succeeded_step_annotation(self, step_name_human) -> str:
+        return bk_annotate(body=f"| {BuildkiteEmojis.success} | {step_name_human} |\n", context=self.group_key, mode="append")
+
+    def emit_command(self, step_name_human, test_command: str) -> str:
+        return LiteralScalarString(f"""
 {self.prepare_shell()}
 # temporarily disable immediate failure on errors, so that we can update the BK annotation
 set +eo pipefail
 {test_command}
 if [[ $$? -ne 0 ]]; then
-  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.failed, context=self.group_key, mode="append")}
+    {self.failed_step_annotation(step_name_human)}
   exit 1
 else
-  {bk_annotate(job_name_human, job_name_slug, self.os, self.jdk, BuildkiteEmojis.success, context=self.group_key, mode="append")}
+    {self.succeeded_step_annotation(step_name_human)}
 fi
       """)
 
-    def java_unit_test(self) -> typing.Tuple[str, str]:
-        job_name_human = "Java Unit Test"
-        job_name_slug = "java-unit-test"
+    def java_unit_test(self) -> JobRetValues:
+        step_name_human = "Java Unit Test"
         test_command = '''
 export ENABLE_SONARQUBE="false"
 ci/unit_tests.sh java
         '''
 
-        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key="java-unit-test",
+            depends=self.init_annotation_key,
+        )
 
-    def ruby_unit_test(self) -> typing.Tuple[str, str]:
-        job_name_human = "Ruby Unit Test"
-        job_name_slug = "ruby-unit-test"
+    def ruby_unit_test(self) -> JobRetValues:
+        step_name_human = "Ruby Unit Test"
+        step_name_key = "ruby-unit-test"
         test_command = """
 ci/unit_tests.sh ruby
         """
 
-        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key=step_name_key,
+            depends=self.init_annotation_key,
+        )
 
-    def integration_tests_part_1(self) -> typing.Tuple[str, str]:
+    def integration_tests_part_1(self) -> JobRetValues:
         return self.integration_tests(part=1)
 
-    def integration_tests_part_2(self) -> typing.Tuple[str, str]:
+    def integration_tests_part_2(self) -> JobRetValues:
         return self.integration_tests(part=2)
 
-    def integration_tests(self, part: int) -> typing.Tuple[str, str]:
-        job_name_human = f"Integration Tests - {part}"
-        job_name_slug = f"integration-tests-pt-{part}"
+    def integration_tests(self, part: int) -> JobRetValues:
+        step_name_human = f"Integration Tests - {part}"
+        step_name_key = f"integration-tests-{part}"
         test_command = f"""
 ci/integration_tests.sh split {part-1}
         """
 
-        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key=step_name_key,
+            depends=self.init_annotation_key,
+        )
 
-    def pq_integration_tests_part_1(self) -> typing.Tuple[str, str]:
+    def pq_integration_tests_part_1(self) -> JobRetValues:
         return self.pq_integration_tests(part=1)
 
-    def pq_integration_tests_part_2(self) -> typing.Tuple[str, str]:
+    def pq_integration_tests_part_2(self) -> JobRetValues:
         return self.pq_integration_tests(part=2)
 
-    def pq_integration_tests(self, part: int) -> typing.Tuple[str, str]:
-        job_name_human = f"IT Persistent Queues - {part}"
-        job_name_slug = f"it-persistent-queues-pt-{part}"
+    def pq_integration_tests(self, part: int) -> JobRetValues:
+        step_name_human = f"IT Persistent Queues - {part}"
+        step_name_key = f"it-persistent-queues-{part}"
         test_command = f"""
 export FEATURE_FLAG=persistent_queues
 ci/integration_tests.sh split {part-1}
         """
 
-        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key=step_name_key,
+            depends=self.init_annotation_key,
+        )
 
-    def x_pack_unit_tests(self) -> typing.Tuple[str, str]:
-        job_name_human = "x-pack unit tests"
-        job_name_slug = "x-pack-unit-tests"
+    def x_pack_unit_tests(self) -> JobRetValues:
+        step_name_human = "x-pack unit tests"
+        step_name_key = "x-pack-unit-test"
         test_command = """
 x-pack/ci/unit_tests.sh
         """
 
-        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key=step_name_key,
+            depends=self.init_annotation_key,
+        )
 
-    def x_pack_integration(self) -> typing.Tuple[str, str]:
-        job_name_human = "x-pack integration"
-        job_name_slug = "x-pack-integration"
+    def x_pack_integration(self) -> JobRetValues:
+        step_name_human = "x-pack integration"
+        step_name_key = "x-pack-integration"
         test_command = """
 x-pack/ci/integration_tests.sh
         """
 
-        return job_name_human, self.emit_command(job_name_human, job_name_slug, test_command)
+        return JobRetValues(
+            step_label=step_name_human,
+            command=test_command,
+            step_key=step_name_key,
+            depends=self.init_annotation_key,
+        )
 
 
 if __name__ == "__main__":
@@ -196,21 +272,27 @@ def x_pack_integration(self) -> typing.Tuple[str, str]:
 
           group_steps = []
           for job in jobs.all_jobs():
-            job_name_human, shell_command = job()
+            job_values = job()
 
             step = {
-              "label": f"{matrix_os} / {matrix_jdk} / {job_name_human}",
-              "agents": {
-                  "provider": "gcp",
-                  "imageProject": "elastic-images-qa",
-                  "image": f"family/platform-ingest-logstash-multi-jdk-{matrix_os}",
-                  "machineType": "n2-standard-4",
-                  "diskSizeGb": 200,
-                  "diskType": "pd-ssd",
-                },
-                "command": shell_command,
+              "label": f"{matrix_os} / {matrix_jdk} / {job_values.step_label}",
+              "key": job_values.step_key,
+              "command": job_values.command,
             }
 
+            if not job_values.default_agent:
+                step["agents"] = {
+                    "provider": "gcp",
+                    "imageProject": "elastic-images-qa",
+                    "image": f"family/platform-ingest-logstash-multi-jdk-{matrix_os}",
+                    "machineType": "n2-standard-4",
+                    "diskSizeGb": 200,
+                    "diskType": "pd-ssd",
+                    }
+
+            if job_values.depends:
+                step["depends_on"] = job_values.depends
+
             group_steps.append(step)
 
 

From c382319f08e367e5d3d19355a400cdc6eb241a59 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 16:57:18 +0200
Subject: [PATCH 07/17] bug fix

---
 .../scripts/jdk-matrix-tests/generate-steps.py     | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index c378e069213..939136343eb 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -52,7 +52,7 @@ def __init__(self, os: str, jdk: str, group_key: str):
         self.os = os
         self.jdk = jdk
         self.group_key = group_key
-        self.init_annotation_key = "initialize-annotation"
+        self.init_annotation_key = f"{os}-{jdk}-initialize-annotation"
 
     def init_annotation(self) -> JobRetValues:
         """
@@ -162,7 +162,7 @@ def java_unit_test(self) -> JobRetValues:
 
         return JobRetValues(
             step_label=step_name_human,
-            command=test_command,
+            command=self.emit_command(step_name_human, test_command),
             step_key="java-unit-test",
             depends=self.init_annotation_key,
         )
@@ -176,7 +176,7 @@ def ruby_unit_test(self) -> JobRetValues:
 
         return JobRetValues(
             step_label=step_name_human,
-            command=test_command,
+            command=self.emit_command(step_name_human, test_command),
             step_key=step_name_key,
             depends=self.init_annotation_key,
         )
@@ -196,7 +196,7 @@ def integration_tests(self, part: int) -> JobRetValues:
 
         return JobRetValues(
             step_label=step_name_human,
-            command=test_command,
+            command=self.emit_command(step_name_human, test_command),
             step_key=step_name_key,
             depends=self.init_annotation_key,
         )
@@ -217,7 +217,7 @@ def pq_integration_tests(self, part: int) -> JobRetValues:
 
         return JobRetValues(
             step_label=step_name_human,
-            command=test_command,
+            command=self.emit_command(step_name_human, test_command),
             step_key=step_name_key,
             depends=self.init_annotation_key,
         )
@@ -231,7 +231,7 @@ def x_pack_unit_tests(self) -> JobRetValues:
 
         return JobRetValues(
             step_label=step_name_human,
-            command=test_command,
+            command=self.emit_command(step_name_human, test_command),
             step_key=step_name_key,
             depends=self.init_annotation_key,
         )
@@ -245,7 +245,7 @@ def x_pack_integration(self) -> JobRetValues:
 
         return JobRetValues(
             step_label=step_name_human,
-            command=test_command,
+            command=self.emit_command(step_name_human, test_command),
             step_key=step_name_key,
             depends=self.init_annotation_key,
         )

From f85f2f9f71cffbcdae34bc454907e1eba7ce0eaf Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 17:08:57 +0200
Subject: [PATCH 08/17] more bug fixes

---
 .../jdk-matrix-tests/generate-steps.py        | 33 ++++++++++---------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 939136343eb..e22c75b5f18 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -155,6 +155,7 @@ def emit_command(self, step_name_human, test_command: str) -> str:
 
     def java_unit_test(self) -> JobRetValues:
         step_name_human = "Java Unit Test"
+        step_key = f"{self.group_key}-java-unit-test"
         test_command = '''
 export ENABLE_SONARQUBE="false"
 ci/unit_tests.sh java
@@ -163,13 +164,13 @@ def java_unit_test(self) -> JobRetValues:
         return JobRetValues(
             step_label=step_name_human,
             command=self.emit_command(step_name_human, test_command),
-            step_key="java-unit-test",
+            step_key=step_key,
             depends=self.init_annotation_key,
         )
 
     def ruby_unit_test(self) -> JobRetValues:
         step_name_human = "Ruby Unit Test"
-        step_name_key = "ruby-unit-test"
+        step_key = f"{self.group_key}-ruby-unit-test"
         test_command = """
 ci/unit_tests.sh ruby
         """
@@ -177,7 +178,7 @@ def ruby_unit_test(self) -> JobRetValues:
         return JobRetValues(
             step_label=step_name_human,
             command=self.emit_command(step_name_human, test_command),
-            step_key=step_name_key,
+            step_key=step_key,
             depends=self.init_annotation_key,
         )
 
@@ -189,7 +190,7 @@ def integration_tests_part_2(self) -> JobRetValues:
 
     def integration_tests(self, part: int) -> JobRetValues:
         step_name_human = f"Integration Tests - {part}"
-        step_name_key = f"integration-tests-{part}"
+        step_key = f"{self.group_key}-integration-tests-{part}"
         test_command = f"""
 ci/integration_tests.sh split {part-1}
         """
@@ -197,7 +198,7 @@ def integration_tests(self, part: int) -> JobRetValues:
         return JobRetValues(
             step_label=step_name_human,
             command=self.emit_command(step_name_human, test_command),
-            step_key=step_name_key,
+            step_key=step_key,
             depends=self.init_annotation_key,
         )
 
@@ -209,7 +210,7 @@ def pq_integration_tests_part_2(self) -> JobRetValues:
 
     def pq_integration_tests(self, part: int) -> JobRetValues:
         step_name_human = f"IT Persistent Queues - {part}"
-        step_name_key = f"it-persistent-queues-{part}"
+        step_key = f"{self.group_key}-it-persistent-queues-{part}"
         test_command = f"""
 export FEATURE_FLAG=persistent_queues
 ci/integration_tests.sh split {part-1}
@@ -218,13 +219,13 @@ def pq_integration_tests(self, part: int) -> JobRetValues:
         return JobRetValues(
             step_label=step_name_human,
             command=self.emit_command(step_name_human, test_command),
-            step_key=step_name_key,
+            step_key=step_key,
             depends=self.init_annotation_key,
         )
 
     def x_pack_unit_tests(self) -> JobRetValues:
         step_name_human = "x-pack unit tests"
-        step_name_key = "x-pack-unit-test"
+        step_key = f"{self.group_key}-x-pack-unit-test"
         test_command = """
 x-pack/ci/unit_tests.sh
         """
@@ -232,13 +233,13 @@ def x_pack_unit_tests(self) -> JobRetValues:
         return JobRetValues(
             step_label=step_name_human,
             command=self.emit_command(step_name_human, test_command),
-            step_key=step_name_key,
+            step_key=step_key,
             depends=self.init_annotation_key,
         )
 
     def x_pack_integration(self) -> JobRetValues:
         step_name_human = "x-pack integration"
-        step_name_key = "x-pack-integration"
+        step_key = f"{self.group_key}-x-pack-integration"
         test_command = """
 x-pack/ci/integration_tests.sh
         """
@@ -246,7 +247,7 @@ def x_pack_integration(self) -> JobRetValues:
         return JobRetValues(
             step_label=step_name_human,
             command=self.emit_command(step_name_human, test_command),
-            step_key=step_name_key,
+            step_key=step_key,
             depends=self.init_annotation_key,
         )
 
@@ -277,9 +278,11 @@ def x_pack_integration(self) -> JobRetValues:
             step = {
               "label": f"{matrix_os} / {matrix_jdk} / {job_values.step_label}",
               "key": job_values.step_key,
-              "command": job_values.command,
             }
 
+            if job_values.depends:
+                step["depends_on"] = job_values.depends
+
             if not job_values.default_agent:
                 step["agents"] = {
                     "provider": "gcp",
@@ -288,10 +291,10 @@ def x_pack_integration(self) -> JobRetValues:
                     "machineType": "n2-standard-4",
                     "diskSizeGb": 200,
                     "diskType": "pd-ssd",
-                    }
+                }
 
-            if job_values.depends:
-                step["depends_on"] = job_values.depends
+
+            step["command"] = job_values.command
 
             group_steps.append(step)
 

From f883936e41699fc21f51c522a6bb17029f174dc4 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 17:35:33 +0200
Subject: [PATCH 09/17] better headers for markdown table

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index e22c75b5f18..006ba739817 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -40,10 +40,11 @@ def get_bk_metadata(key: str) -> typing.List[str]:
         exit(1)
 
 def bk_annotate(body: str, context: str, mode = "") -> str:
-    cmd = f"""buildkite-agent annotate "{body}" --context={context}"""
+    cmd = f"""buildkite-agent annotate --context={context} """
     if mode:
-        cmd += f" --{mode}"
+        cmd += f"--{mode} "
 
+    cmd += f"\"{body}\"\n"
     return cmd
 
 
@@ -59,10 +60,7 @@ def init_annotation(self) -> JobRetValues:
         Command for creating the header of a new annotation for a group step
         """
 
-        body = f'''## Group {self.os} / {self.jdk}
-
-        | **Status** | **Test** |
-        '''
+        body = f"## Group {self.os} / {self.jdk}\n| **Status** | **Test** |"
 
         return JobRetValues(
             step_label="Initialize annotation",

From 63215882a3dee40f1ed445c3bf568fecdf9c2a32 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 18:11:07 +0200
Subject: [PATCH 10/17] fix newlines in annotations

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 006ba739817..52dd37d41e3 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -44,7 +44,7 @@ def bk_annotate(body: str, context: str, mode = "") -> str:
     if mode:
         cmd += f"--{mode} "
 
-    cmd += f"\"{body}\"\n"
+    cmd += f"\"{body}\n\""
     return cmd
 
 
@@ -132,10 +132,10 @@ def prepare_shell(self) -> str:
 """
 
     def failed_step_annotation(self, step_name_human) -> str:
-        return bk_annotate(body=f"| {BuildkiteEmojis.failed} | {step_name_human} |\n", context=self.group_key, mode="append")
+        return bk_annotate(body=f"| {BuildkiteEmojis.failed} | {step_name_human} |", context=self.group_key, mode="append")
 
     def succeeded_step_annotation(self, step_name_human) -> str:
-        return bk_annotate(body=f"| {BuildkiteEmojis.success} | {step_name_human} |\n", context=self.group_key, mode="append")
+        return bk_annotate(body=f"| {BuildkiteEmojis.success} | {step_name_human} |", context=self.group_key, mode="append")
 
     def emit_command(self, step_name_human, test_command: str) -> str:
         return LiteralScalarString(f"""

From 3cd146354bc675645c047beabbf14a48ead7e685 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 18:18:39 +0200
Subject: [PATCH 11/17] more formatting

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 52dd37d41e3..743f30fe0ec 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -44,7 +44,7 @@ def bk_annotate(body: str, context: str, mode = "") -> str:
     if mode:
         cmd += f"--{mode} "
 
-    cmd += f"\"{body}\n\""
+    cmd += f"\"\n{body}\n\""
     return cmd
 
 
@@ -60,7 +60,7 @@ def init_annotation(self) -> JobRetValues:
         Command for creating the header of a new annotation for a group step
         """
 
-        body = f"## Group {self.os} / {self.jdk}\n| **Status** | **Test** |"
+        body = f"### Group {self.os} / {self.jdk}\n| **Status** | **Test** |"
 
         return JobRetValues(
             step_label="Initialize annotation",

From 9fe362b0a248b06434df8c7e6b8c6c2c711fcf15 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 18:19:33 +0200
Subject: [PATCH 12/17] use info for annotation

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 743f30fe0ec..3790ca4c0f1 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -40,7 +40,7 @@ def get_bk_metadata(key: str) -> typing.List[str]:
         exit(1)
 
 def bk_annotate(body: str, context: str, mode = "") -> str:
-    cmd = f"""buildkite-agent annotate --context={context} """
+    cmd = f"""buildkite-agent annotate --style=info --context={context} """
     if mode:
         cmd += f"--{mode} "
 

From 70cf2cf1c7b449a23be630de0b6b4385b7ede966 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 18:30:58 +0200
Subject: [PATCH 13/17] fix table

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 3790ca4c0f1..b5088d11845 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -60,7 +60,7 @@ def init_annotation(self) -> JobRetValues:
         Command for creating the header of a new annotation for a group step
         """
 
-        body = f"### Group {self.os} / {self.jdk}\n| **Status** | **Test** |"
+        body = f"### Group {self.os} / {self.jdk}\n| **Status** | **Test** |\n| --- | ----|"
 
         return JobRetValues(
             step_label="Initialize annotation",

From c102882eba36f0076f92c84a8383d6db46e71299 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 18:41:22 +0200
Subject: [PATCH 14/17] improved md table formatting

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index b5088d11845..69fa5a880d8 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -44,7 +44,7 @@ def bk_annotate(body: str, context: str, mode = "") -> str:
     if mode:
         cmd += f"--{mode} "
 
-    cmd += f"\"\n{body}\n\""
+    cmd += f"\"\n{body}\""
     return cmd
 
 

From d775ea6ea50ad0ca3a329b525f326c4090715f47 Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 18:59:38 +0200
Subject: [PATCH 15/17] final improvements to md format

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 69fa5a880d8..da81f901471 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -44,7 +44,7 @@ def bk_annotate(body: str, context: str, mode = "") -> str:
     if mode:
         cmd += f"--{mode} "
 
-    cmd += f"\"\n{body}\""
+    cmd += f"\"{body}\n\""
     return cmd
 
 
@@ -60,7 +60,7 @@ def init_annotation(self) -> JobRetValues:
         Command for creating the header of a new annotation for a group step
         """
 
-        body = f"### Group {self.os} / {self.jdk}\n| **Status** | **Test** |\n| --- | ----|"
+        body = f"### Group: `{self.os} / {self.jdk}`\n| **Status** | **Test** |\n| --- | ----|"
 
         return JobRetValues(
             step_label="Initialize annotation",

From b89a941c6a9462c82d1954a5000b09b341e8fd8c Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 19:34:00 +0200
Subject: [PATCH 16/17] Remove back ticks

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index da81f901471..5016a0dc036 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -60,7 +60,7 @@ def init_annotation(self) -> JobRetValues:
         Command for creating the header of a new annotation for a group step
         """
 
-        body = f"### Group: `{self.os} / {self.jdk}`\n| **Status** | **Test** |\n| --- | ----|"
+        body = f"### Group: {self.os} / {self.jdk}\n| **Status** | **Test** |\n| --- | ----|"
 
         return JobRetValues(
             step_label="Initialize annotation",

From 6cefd3cf69d53eb3f2be5a61261034370b6466db Mon Sep 17 00:00:00 2001
From: Dimitrios Liappis <dimitrios.liappis@gmail.com>
Date: Wed, 8 Nov 2023 20:24:09 +0200
Subject: [PATCH 17/17] Uncomment out type hint

---
 .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
index 5016a0dc036..6df12f3c75b 100644
--- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
+++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py
@@ -79,7 +79,7 @@ class WindowsJobs(Jobs):
     def __init__(self, os: str, jdk: str, group_key: str):
       super().__init__(os=os, jdk=jdk, group_key=group_key)
 
-    def all_jobs(self):# -> list[Callable[[], JobRetValues]]:
+    def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]:
         return [
           self.unit_tests,
         ]
@@ -101,7 +101,7 @@ class LinuxJobs(Jobs):
     def __init__(self, os: str, jdk: str, group_key: str):
       super().__init__(os=os, jdk=jdk, group_key=group_key)
 
-    def all_jobs(self):# -> list[Callable[[], JobRetValues]]:
+    def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]:
         return [
             self.init_annotation,
             self.java_unit_test,