From 1bb2cf5a0138789b7cbd02834247cb39868d5043 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 10 Nov 2023 14:32:50 +0200 Subject: [PATCH 1/4] [ci] JDK matrix Buildkite pipelines (pt 2/Windows) This commit adds JDK matrix Buildkite pipelines for Windows 2022, 2019 and 2016. It also makes the groups easier to read (on both Linux and Windows pipelines) by removing the os-jdk prefix from the job labels. `testDLQWriterFlusherRemovesExpiredSegmentWhenCurrentHeadSegmentIsEmpty` fails on Windows Buildkite agents and it's a test issue tracked in https://github.com/elastic/logstash/issues/15562. Relates: - https://github.com/elastic/logstash/pull/15539 - https://github.com/elastic/ingest-dev/issues/1725 - --- .../jdk-matrix-tests/generate-steps.py | 25 ++++--- .../jdk-matrix-tests/launch-command.ps1 | 54 +++++++++++++++ .buildkite/windows_jdk_matrix_pipeline.yml | 66 ++++++++++++++++++- 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 .buildkite/scripts/jdk-matrix-tests/launch-command.ps1 diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py index 6df12f3c75b..2a14ddfb3eb 100644 --- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py +++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py @@ -1,5 +1,5 @@ import abc -from dataclasses import dataclass +from dataclasses import dataclass, field import os import sys import typing @@ -14,8 +14,10 @@ class JobRetValues: command: str step_key: str depends: str + artifact_paths: list = field(default_factory=list) default_agent: bool = False + @dataclass class BuildkiteEmojis: running: str = ":bk-status-running:" @@ -81,21 +83,24 @@ def __init__(self, os: str, jdk: str, group_key: str): def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]: return [ + self.init_annotation, self.unit_tests, ] def unit_tests(self) -> JobRetValues: - step_name_human = "Java Unit Test" - test_command = "# TODO" + step_name_human = "Unit Test" + step_key = f"{self.group_key}-unit-test" + test_command = rf'''.\\.buildkite\\scripts\\jdk-matrix-tests\\launch-command.ps1 -JDK "{self.jdk}" -StepNameHuman "{step_name_human}" -AnnotateContext "{self.group_key}" -CIScript ".\\ci\\unit_tests.bat" -Annotate + ''' return JobRetValues( step_label=step_name_human, - command=test_command, - step_key="java-unit-test", - depends="", + command=LiteralScalarString(test_command), + step_key=step_key, + depends=self.init_annotation_key, + artifact_paths=["build_reports.zip"], ) - return step_name_human, test_command - + class LinuxJobs(Jobs): def __init__(self, os: str, jdk: str, group_key: str): @@ -274,7 +279,7 @@ def x_pack_integration(self) -> JobRetValues: job_values = job() step = { - "label": f"{matrix_os} / {matrix_jdk} / {job_values.step_label}", + "label": f"{job_values.step_label}", "key": job_values.step_key, } @@ -291,6 +296,8 @@ def x_pack_integration(self) -> JobRetValues: "diskType": "pd-ssd", } + if job_values.artifact_paths: + step["artifact_paths"] = job_values.artifact_paths step["command"] = job_values.command diff --git a/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 b/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 new file mode 100644 index 00000000000..5b144e7e8ef --- /dev/null +++ b/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 @@ -0,0 +1,54 @@ +# ******************************************************** +# This file contains prerequisite bootstrap invocations +# required for Logstash CI JDK matrix tests +# ******************************************************** + +param ( + [string]$JDK, + [string]$CIScript, + [string]$StepNameHuman, + [string]$AnnotateContext, + [switch]$Annotate +) + +# expand previous buildkite folded section (command invocation) +Write-Host "^^^ +++" + +# the unit test script expects the WORKSPACE env var +$env:WORKSPACE = $PWD.Path + +# unset generic JAVA_HOME +if (Test-Path env:JAVA_HOME) { + Remove-Item -Path env:JAVA_HOME + Write-Host "--- Environment variable 'JAVA_HOME' has been unset." +} else { + Write-Host "--- Environment variable 'JAVA_HOME' doesn't exist. Continuing." +} + +# LS env vars for JDK matrix tests +$JAVA_CUSTOM_DIR = "C:\Users\buildkite\.java\$JDK" +$env:BUILD_JAVA_HOME = $JAVA_CUSTOM_DIR +$env:RUNTIME_JAVA_HOME = $JAVA_CUSTOM_DIR +$env:LS_JAVA_HOME = $JAVA_CUSTOM_DIR + +Write-Host "--- Running test: $CIScript" +try { + Invoke-Expression $CIScript + + if ($LASTEXITCODE -ne 0) { + throw "Test script $CIScript failed with a non-zero code: $LASTEXITCODE" + } + + if ($Annotate) { + C:\buildkite-agent\bin\buildkite-agent.exe annotate --context="$AnnotateContext" --append "| :bk-status-passed: | $StepNameHuman |`n" + } +} catch { + # tests failed + Write-Host "^^^ +++" + if ($Annotate) { + C:\buildkite-agent\bin\buildkite-agent.exe annotate --context="$AnnotateContext" --append "| :bk-status-failed: | $StepNameHuman |`n" + Write-Host "--- Archiving test reports" + & "7z.exe" a -r .\build_reports.zip .\logstash-core\build\reports\tests + } + exit 1 +} diff --git a/.buildkite/windows_jdk_matrix_pipeline.yml b/.buildkite/windows_jdk_matrix_pipeline.yml index e7fadf31279..88c55a094fb 100644 --- a/.buildkite/windows_jdk_matrix_pipeline.yml +++ b/.buildkite/windows_jdk_matrix_pipeline.yml @@ -1,3 +1,65 @@ steps: - - label: "Test Windows JDK matrix pipeline" - command: "echo 'Hello world'" + - input: "Test Parameters" + if: build.source != "schedule" + fields: + - select: "Operating System" + key: "matrix-os" + hint: "The operating system variant(s) to run on:" + required: true + multiple: true + default: "windows-2022" + options: + - label: "Windows 2022" + value: "windows-2022" + - label: "Windows 2019" + value: "windows-2019" + - label: "Windows 2016" + value: "windows-2016" + + - select: "Java" + key: "matrix-jdk" + hint: "The JDK to test with:" + required: true + multiple: true + default: "adoptiumjdk_17" + options: + - label: "Adoptium JDK 17 (Eclipse Temurin)" + value: "adoptiumjdk_17" + - label: "Adopt OpenJDK 11" + value: "adoptopenjdk_11" + - label: "OpenJDK 17" + value: "openjdk_17" + - label: "OpenJDK 11" + value: "openjdk_11" + - label: "Zulu 17" + value: "zulu_17" + - label: "Zulu 11" + value: "zulu_11" + + - wait: ~ + if: build.source != "schedule" + + - command: | + set -euo pipefail + + echo "--- Downloading prerequisites" + python3 -m pip install ruamel.yaml + + echo "--- Printing generated dynamic steps" + export MATRIX_OSES="$(buildkite-agent meta-data get matrix-os)" + export MATRIX_JDKS="$(buildkite-agent meta-data get matrix-jdk)" + set +eo pipefail + python3 .buildkite/scripts/jdk-matrix-tests/generate-steps.py >pipeline_steps.yml + if [[ $$? -ne 0 ]]; then + echo "^^^ +++" + echo "There was a problem rendering the pipeline steps." + cat pipeline_steps.yml + echo "Exiting now." + exit 1 + else + set -eo pipefail + cat pipeline_steps.yml + fi + + echo "--- Uploading steps to buildkite" + cat pipeline_steps.yml | buildkite-agent pipeline upload From 78657e2a72d53d6c4f9cbc72cda4e82d94c09cc0 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 10 Nov 2023 14:47:50 +0200 Subject: [PATCH 2/4] Test making DLQ test succeed on Windows --- .../common/io/DeadLetterQueueWriterAgeRetentionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java b/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java index 6c9bb5a024c..b786a7b5ce6 100644 --- a/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java +++ b/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java @@ -337,7 +337,7 @@ public void testDLQWriterFlusherRemovesExpiredSegmentWhenCurrentHeadSegmentIsEmp DLQEntry entry = new DLQEntry(event, "", "", "00001", DeadLetterQueueReaderTest.constantSerializationLengthTimestamp(fakeClock)); writeManager.writeEntry(entry); - + Thread.sleep(101); // wait the flush interval so that the current head segment is sealed Awaitility.await("After the flush interval head segment is sealed and a fresh empty head is created") .atLeast(flushInterval) From 491fed51ecb017b7b47460045f3078f393abcb02 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 10 Nov 2023 16:57:21 +0200 Subject: [PATCH 3/4] Improve unit test label on Windows --- .buildkite/scripts/jdk-matrix-tests/generate-steps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py index 2a14ddfb3eb..d757a42491c 100644 --- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py +++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py @@ -88,7 +88,7 @@ def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]: ] def unit_tests(self) -> JobRetValues: - step_name_human = "Unit Test" + step_name_human = "Unit Test (Java/Ruby)" step_key = f"{self.group_key}-unit-test" test_command = rf'''.\\.buildkite\\scripts\\jdk-matrix-tests\\launch-command.ps1 -JDK "{self.jdk}" -StepNameHuman "{step_name_human}" -AnnotateContext "{self.group_key}" -CIScript ".\\ci\\unit_tests.bat" -Annotate ''' From edcdca9cb6b3132fadc5ac69de3e6de1e6726863 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 10 Nov 2023 17:00:24 +0200 Subject: [PATCH 4/4] don't touch test --- .../common/io/DeadLetterQueueWriterAgeRetentionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java b/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java index b786a7b5ce6..6c9bb5a024c 100644 --- a/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java +++ b/logstash-core/src/test/java/org/logstash/common/io/DeadLetterQueueWriterAgeRetentionTest.java @@ -337,7 +337,7 @@ public void testDLQWriterFlusherRemovesExpiredSegmentWhenCurrentHeadSegmentIsEmp DLQEntry entry = new DLQEntry(event, "", "", "00001", DeadLetterQueueReaderTest.constantSerializationLengthTimestamp(fakeClock)); writeManager.writeEntry(entry); - Thread.sleep(101); + // wait the flush interval so that the current head segment is sealed Awaitility.await("After the flush interval head segment is sealed and a fresh empty head is created") .atLeast(flushInterval)