Fix IT tests after version bumps

This commit fixes IT failures that frequently occur after
version bumps due to missing unified release snapshot builds for
the new version.

This commit uses project specific DRA snapshot URLs for ES and Filebeat
in all cases apart from release builds.

Closes #2825

build.gradle

@@ -51,6 +51,7 @@ import org.logstash.gradle.tooling.ListProjectDependencies
 import org.logstash.gradle.tooling.ExtractBundledJdkVersion
 import org.logstash.gradle.tooling.SignAliasDefinitions
 import org.logstash.gradle.tooling.ToolingUtils
+import org.logstash.gradle.tooling.SnapshotArtifactURLs
 
 allprojects {
   group = 'org.logstash'
@@ -180,6 +181,11 @@ tasks.register("configureArtifactInfo") {
         def dlVersions = new JsonSlurper().parseText(apiResponse)
         String qualifiedVersion = dlVersions['versions'].grep(isReleaseBuild ? ~/^${version}$/ : ~/^${version}-SNAPSHOT/)[0]
         if (qualifiedVersion == null) {
+            if (isReleaseBuild == false) {
+                project.ext.set("useProjectSpecificArtifactSnapshotUrl", true)
+                project.ext.set("stackArtifactSuffix", "${version}-SNAPSHOT")
+                return
+            }
             throw new GradleException("could not find the current artifact from the artifact-api ${artifactVersionsApi} for ${version}")
         }
         // find latest reference to last build
@@ -190,6 +196,7 @@ tasks.register("configureArtifactInfo") {
 
         project.ext.set("artifactApiVersionedBuildUrl", "${artifactVersionsApi}/${qualifiedVersion}/builds/${stackBuildVersion}")
         project.ext.set("stackArtifactSuffix", qualifiedVersion)
+        project.ext.set("useProjectSpecificArtifactSnapshotUrl", false)
     }
 }
 
@@ -432,16 +439,24 @@ tasks.register("downloadFilebeat") {
 
     doLast {
         download {
-            String downloadedFilebeatName = "filebeat-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("beatsArchitecture")}"
+            String beatVersion = project.ext.get("stackArtifactSuffix")
+            String downloadedFilebeatName = "filebeat-${beatVersion}-${project.ext.get("beatsArchitecture")}"
             project.ext.set("unpackedFilebeatName", downloadedFilebeatName)
 
-            // find url of build artifact
-            String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
-            String apiResponse = artifactApiUrl.toURL().text
-            def buildUrls = new JsonSlurper().parseText(apiResponse)
-
-            project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
-            project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
+            if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
+                List packageUrls = SnapshotArtifactURLs.packageUrls("beats", beatVersion, downloadedFilebeatName)
+                project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: packageUrls[0])
+                project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
+            }
+            else {
+                // find url of build artifact
+                String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/beats/packages/${downloadedFilebeatName}.tar.gz"
+                String apiResponse = artifactApiUrl.toURL().text
+                def buildUrls = new JsonSlurper().parseText(apiResponse)
+
+                project.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
+                project.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
+            }
 
             src project.ext.filebeatSnapshotUrl
             onlyIfNewer true
@@ -477,14 +492,21 @@ tasks.register("checkEsSHA") {
     description "Download ES version remote's fingerprint file"
 
     doLast {
-        String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
-
+        String esVersion = project.ext.get("stackArtifactSuffix")
+        String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
+        String remoteSHA
 
-        // find url of build artifact
-        String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
-        String apiResponse = artifactApiUrl.toURL().text
-        def buildUrls = new JsonSlurper().parseText(apiResponse)
-        String remoteSHA = buildUrls.package.sha_url.toURL().text
+        if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
+            List packageUrls = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
+            remoteSHA = packageUrls[1]
+        }
+        else {
+            // find url of build artifact
+            String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
+            String apiResponse = artifactApiUrl.toURL().text
+            def buildUrls = new JsonSlurper().parseText(apiResponse)
+            remoteSHA = buildUrls.package.sha_url.toURL().text
+        }
 
         def localESArchive = new File("${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
         if (localESArchive.exists()) {
@@ -512,22 +534,32 @@ tasks.register("checkEsSHA") {
 
 tasks.register("downloadEs") {
     dependsOn  = [configureArtifactInfo, checkEsSHA]
-    description "Download ES Snapshot for current branch version: ${version}"
 
+    description "Download ES Snapshot for current branch version: ${version}"
     inputs.file("${projectDir}/versions.yml")
 
     doLast {
         download {
-            String downloadedElasticsearchName = "elasticsearch-${project.ext.get("stackArtifactSuffix")}-${project.ext.get("esArchitecture")}"
-            project.ext.set("unpackedElasticsearchName", "elasticsearch-${project.ext.get("stackArtifactSuffix")}")
+            String esVersion = project.ext.get("stackArtifactSuffix")
+            String downloadedElasticsearchName = "elasticsearch-${esVersion}-${project.ext.get("esArchitecture")}"
 
-            // find url of build artifact
-            String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
-            String apiResponse = artifactApiUrl.toURL().text
-            def buildUrls = new JsonSlurper().parseText(apiResponse)
+            project.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
+
+            if (project.ext.get("useProjectSpecificArtifactSnapshotUrl")) {
+                List packageUrls = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
+                project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: packageUrls[0])
+                project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+            }
+            else {
+                // find url of build artifact
+                String artifactApiUrl = "${project.ext.get("artifactApiVersionedBuildUrl")}/projects/elasticsearch/packages/${downloadedElasticsearchName}.tar.gz"
+                String apiResponse = artifactApiUrl.toURL().text
+
+                def buildUrls = new JsonSlurper().parseText(apiResponse)
 
-            project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
-            project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+                project.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: buildUrls["package"]["url"])
+                project.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
+            }
 
             src project.ext.elasticsearchSnapshotURL
             onlyIfNewer true

buildSrc/src/main/groovy/org/logstash/gradle/tooling/SnapshotArtifactURLs.groovy

@@ -0,0 +1,29 @@
+package org.logstash.gradle.tooling
+
+import groovy.json.JsonSlurper
+
+/**
+ * Helper class to obtain project specific (e.g. Elasticsearch or beats/filebeat) snapshot-DRA artifacts.
+ * We use it in all cases apart from release builds.
+ * */
+class SnapshotArtifactURLs {
+    /**
+     * Returns a list of the package and package SHA(512) URLs for a given project / version / downloadedPackageName
+     * */
+    static def packageUrls(String project, String projectVersion, String downloadedPackageName) {
+        String artifactSnapshotVersionsApiPrefix = "https://artifacts-snapshot.elastic.co"
+
+        // e.g. https://artifacts-snapshot.elastic.co/elasticsearch/latest/8.11.5-SNAPSHOT.json
+        String apiResponse = "${artifactSnapshotVersionsApiPrefix}/${project}/latest/${projectVersion}.json".toURL().text
+        def artifactUrls = new JsonSlurper().parseText(apiResponse)
+        String manifestUrl = artifactUrls["manifest_url"]
+        // e.g. https://artifacts-snapshot.elastic.co/elasticsearch/8.11.5-12345678/manifest-8.11.5-SNAPSHOT.json
+        apiResponse = manifestUrl.toURL().text
+        def packageArtifactUrls = new JsonSlurper().parseText(apiResponse)
+        String packageUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["url"]
+        String packageShaUrl = packageArtifactUrls["projects"]["${project}"]["packages"]["${downloadedPackageName}.tar.gz"]["sha_url"]
+
+        return [packageUrl, packageShaUrl]
+
+    }
+}