From ef8451ea3de1dc65f60e34aff3d6ef2c67c490dd Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 1 Feb 2024 10:04:25 +0200 Subject: [PATCH] Allow running Java+Ruby tests on Windows separately (#15861) This commit allows separate running of Java and Ruby tests on Windows i.e. the same way as we currently do on unix (unit_tests.sh) via a cli argument. If no argument has been supplied, both tests are run (as it does now). The wrapper script is also rewritten from old batch style script to Powershell. This work allows us to split the existing Windows CI job in a subsequent PR to separate steps, as we currently do on Linux. Relates: https://github.com/elastic/logstash/issues/15566 (cherry picked from commit 8ac55184b8a9d22dea8f61f287a3cd21440fda39) --- .../exhaustive-tests/generate-steps.py | 2 +- .../jdk-matrix-tests/generate-steps.py | 2 +- .../jdk-matrix-tests/launch-command.ps1 | 3 - ci/unit_tests.bat | 56 --------- ci/unit_tests.ps1 | 106 ++++++++++++++++++ 5 files changed, 108 insertions(+), 61 deletions(-) delete mode 100644 ci/unit_tests.bat create mode 100644 ci/unit_tests.ps1 diff --git a/.buildkite/scripts/exhaustive-tests/generate-steps.py b/.buildkite/scripts/exhaustive-tests/generate-steps.py index 95096a2340b..8944a6e1e2d 100644 --- a/.buildkite/scripts/exhaustive-tests/generate-steps.py +++ b/.buildkite/scripts/exhaustive-tests/generate-steps.py @@ -38,7 +38,7 @@ def compat_linux_step(imagesuffix: str) -> dict[str, typing.Any]: def compat_windows_step(imagesuffix: str) -> dict[str, typing.Any]: - windows_command = LiteralScalarString(r'''$$env:WORKSPACE=$$PWD.Path ; .\\ci\\unit_tests.bat''') + windows_command = LiteralScalarString(r'''.\\ci\\unit_tests.ps1''') return compat_step(imagesuffix, command=windows_command) diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py index ab8c2cd43cf..2f9fd7e9f23 100644 --- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py +++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py @@ -141,7 +141,7 @@ def all_jobs(self) -> list[typing.Callable[[], JobRetValues]]: def unit_tests(self) -> JobRetValues: step_name_human = "Unit Test (Java/Ruby)" step_key = f"{self.group_key}-unit-test" - test_command = rf'''.\\.buildkite\\scripts\\jdk-matrix-tests\\launch-command.ps1 -JDK "{self.jdk}" -StepNameHuman "{step_name_human}" -AnnotateContext "{self.group_key}" -CIScript ".\\ci\\unit_tests.bat" -Annotate + test_command = rf'''.\\.buildkite\\scripts\\jdk-matrix-tests\\launch-command.ps1 -JDK "{self.jdk}" -StepNameHuman "{step_name_human}" -AnnotateContext "{self.group_key}" -CIScript ".\\ci\\unit_tests.ps1" -Annotate ''' return JobRetValues( diff --git a/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 b/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 index 5b144e7e8ef..4645881f501 100644 --- a/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 +++ b/.buildkite/scripts/jdk-matrix-tests/launch-command.ps1 @@ -14,9 +14,6 @@ param ( # expand previous buildkite folded section (command invocation) Write-Host "^^^ +++" -# the unit test script expects the WORKSPACE env var -$env:WORKSPACE = $PWD.Path - # unset generic JAVA_HOME if (Test-Path env:JAVA_HOME) { Remove-Item -Path env:JAVA_HOME diff --git a/ci/unit_tests.bat b/ci/unit_tests.bat deleted file mode 100644 index 5d785865728..00000000000 --- a/ci/unit_tests.bat +++ /dev/null @@ -1,56 +0,0 @@ -@echo off -setlocal enabledelayedexpansion - -if "%WORKSPACE%" == "" ( - echo Error: environment variable WORKSPACE must be defined. Aborting.. - exit /B 1 -) - -:: see if %WORKSPACE% is already mapped to a drive -for /f "tokens=1* delims==> " %%G IN ('subst') do ( - set sdrive=%%G - :: removing extra space - set sdrive=!sdrive:~0,2! - set spath=%%H - - if /I "!spath!" == "%WORKSPACE%" ( - set use_drive=!sdrive! - goto :found_drive - ) -) - -:: no existing mapping -:: try to assign "%WORKSPACE%" to the first drive letter which works -for %%i in (A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z) do ( - set "drive=%%i:" - subst !drive! "%WORKSPACE%" >nul - if not errorlevel 1 ( - set use_drive=!drive! - goto :found_drive - ) -) - -echo Error: unable to subst drive to path %WORKSPACE%. Aborting... -exit /B 1 - -:found_drive -echo Using drive !use_drive! for %WORKSPACE% - -:: change current directory to that drive -!use_drive! - -echo Running core tests.. -if defined BUILD_JAVA_HOME ( - if defined GRADLE_OPTS ( - set GRADLE_OPTS=%GRADLE_OPTS% -Dorg.gradle.java.home=%BUILD_JAVA_HOME% - ) else ( - set GRADLE_OPTS=-Dorg.gradle.java.home=%BUILD_JAVA_HOME% - ) -) -echo Invoking Gradle, GRADLE_OPTS: %GRADLE_OPTS%, BUILD_JAVA_HOME: %BUILD_JAVA_HOME% -call .\gradlew.bat test --console=plain --no-daemon --info - -if errorlevel 1 ( - echo Error: failed to run core tests. Aborting.. - exit /B 1 -) diff --git a/ci/unit_tests.ps1 b/ci/unit_tests.ps1 new file mode 100644 index 00000000000..b070c61ffe7 --- /dev/null +++ b/ci/unit_tests.ps1 @@ -0,0 +1,106 @@ +<# +.SYNOPSIS + This is a gradle wrapper script to help run the Logstash unit tests on Windows. + +.PARAMETER UnnamedArgument1 + Optionally allows to specify a subset of tests. + Allows values are "ruby" or "java". + If unset, all tests are executed. + +.EXAMPLE + .\ci\unit_tests.ps1 + Runs all unit tests. + + .\ci\unit_tests.ps1 java + Runs only Java unit tests. +#> + +$selectedTestSuite="all" + +if ($args.Count -eq 1) { + $selectedTestSuite=$args[0] +} + +$startingPath = Get-Location + +## Map a drive letter to the current path to avoid path length issues + +# First, check if there is already a mapping +$currentDir = $PWD.Path +$substOutput = subst + +# Filter the subst output based on the path +$matchedLines = $substOutput | Where-Object { $_ -like "*$currentDir*" } + +if ($matchedLines) { + # $currentDir seems to be already mapped to another drive letter; switch to this drive + # Extract the drive letter from the matched lines + $driveLetter = $matchedLines | ForEach-Object { + # Split the line by colon and extract the drive letter + ($_ -split ':')[0] + } + $drivePath = "$driveLetter`:" + + Write-Output "$currentDir is already mapped to $drivePath." + Set-Location -Path $drivePath + Write-Output "Changing drive to $drivePath." +} +else { + # $currentDir isn't mapped to a drive letter, let's find a free drive letter and change to it + + # Loop through drive letters A to Z; we don't use the 'A'..'Z' for BWC with Windows 2016 / Powershell < 7 + for ($driveLetterAscii = 65; $driveLetterAscii -le 90; $driveLetterAscii++) { + $drivePath = [char]$driveLetterAscii + ":" + + # check if the drive letter is available + if (-not (Test-Path $drivePath)) { + # found a free drive letter, create the virtual drive mapping and switch to it + subst $drivePath $currentDir + + Write-Output "Mapped $currentDir to $drivePath" + Set-Location -Path $drivePath + Write-Output "Changing drive to $drivePath." + # exit the loop since we found a free drive letter + break + } + } +} + +if (Test-Path Env:BUILD_JAVA_HOME) { + if (Test-Path Env:GRADLE_OPTS) { + $env:GRADLE_OPTS=$env:GRADLE_OPTS + " " + "-Dorg.gradle.java.home=" + $env:BUILD_JAVA_HOME + } else { + $env:GRADLE_OPTS="-Dorg.gradle.java.home=" + $env:BUILD_JAVA_HOME + } +} + +$testOpts = "GRADLE_OPTS: $env:GRADLE_OPTS, BUILD_JAVA_HOME: $env:BUILD_JAVA_HOME" + +try { + if ($selectedTestSuite -eq "java") { + Write-Host "~~~ :java: Running Java tests via Gradle using $testOpts" + $CIScript = ".\gradlew.bat javaTests --console=plain --no-daemon --info" + Invoke-Expression $CIScript + } + elseif ($selectedTestSuite -eq "ruby") { + Write-Host "~~~ :ruby: Running Ruby tests via Gradle using $testOpts" + $CIScript = ".\gradlew.bat rubyTests --console=plain --no-daemon --info" + Invoke-Expression $CIScript + } + else { + Write-Host "~~~ Running all tests via Gradle using $testOpts" + $CIScript = ".\gradlew.bat test --console=plain --no-daemon --info" + Invoke-Expression $CIScript + } + + if ($LASTEXITCODE -ne 0) { + throw "Test script $CIScript failed with a non-zero code: $LASTEXITCODE" + } +} catch { + # tests failed + Write-Host "^^^ +++" + exit 1 +} + +# switch back to the path when the script started +Set-Location -Path $startingPath