diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 602004145..8926c401c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - java_version: [19] + java_version: [21] steps: - name: Environment @@ -66,7 +66,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{secrets.TOWER_CI_AWS_SECRET}} DOCKER_USER: ${{ secrets.DOCKER_USER }} DOCKER_PAT: ${{ secrets.DOCKER_PAT }} - QUAY_USER: ${{ secrets.QUAY_USER }} + QUAY_USER: "pditommaso+wave_ci_tests" QUAY_PAT: ${{ secrets.QUAY_PAT }} AZURECR_USER: ${{ secrets.AZURECR_USER }} AZURECR_PAT: ${{ secrets.AZURECR_PAT }} diff --git a/.github/workflows/security-submit-dependecy-graph.yml b/.github/workflows/security-submit-dependecy-graph.yml new file mode 100644 index 000000000..e2c3a7aa7 --- /dev/null +++ b/.github/workflows/security-submit-dependecy-graph.yml @@ -0,0 +1,24 @@ +name: Generate and submit dependency graph for wave +on: + push: + branches: ['master'] + +permissions: + contents: write + +jobs: + dependency-submission: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + + - name: Generate and submit dependency graph for wave + uses: gradle/actions/dependency-submission@v4 + with: + dependency-resolution-task: "dependencies" + additional-arguments: "--configuration runtimeClasspath" + dependency-graph: generate-and-submit diff --git a/Makefile b/Makefile index 5fb8134bc..d5efcd355 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -config ?= compileClasspath +config ?= runtimeClasspath ifdef module mm = :${module}: diff --git a/README.md b/README.md index 10d853924..13acb4415 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ images. * Push and cache built containers to a user-provided container repository; * Build Singularity native containers both using a Singularity spec file, Conda package(s); * Push Singularity native container images to OCI-compliant registries; - +* Scan container images for security vulnerabilities ### How it works @@ -34,7 +34,7 @@ container registry where the image is stored, while the instrumented layers are ### Requirements -* Java 19 or later +* Java 21 or later * Linux or macOS * Redis 6.2 (or later) * Docker engine (for development) diff --git a/VERSION b/VERSION index 80138e714..141f2e805 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.13.4 +1.15.0 diff --git a/build.gradle b/build.gradle index c47b7649a..92edfd34b 100644 --- a/build.gradle +++ b/build.gradle @@ -2,10 +2,10 @@ import java.time.OffsetDateTime import java.time.format.DateTimeFormatter plugins { - id 'java-library' + id 'io.seqera.wave.java-library-conventions' id 'io.seqera.wave.groovy-application-conventions' - id "com.github.johnrengelman.shadow" version "7.1.1" - id "io.micronaut.minimal.application" version "3.7.0" + id "com.github.johnrengelman.shadow" version "8.1.1" + id "io.micronaut.minimal.application" version "4.1.1" id "com.google.cloud.tools.jib" version "3.4.2" id 'org.asciidoctor.jvm.convert' version '3.3.2' id 'jacoco' @@ -29,73 +29,81 @@ repositories { } dependencies { - annotationProcessor("io.micronaut:micronaut-http-validation") - compileOnly("io.micronaut.data:micronaut-data-processor") - compileOnly("io.micronaut:micronaut-inject-groovy") - compileOnly("io.micronaut:micronaut-http-validation") - implementation("jakarta.persistence:jakarta.persistence-api:3.0.0") - api 'io.seqera:lib-mail:1.0.0' - api 'io.seqera:wave-api:0.13.3' - api 'io.seqera:wave-utils:0.14.1' - implementation("io.micronaut:micronaut-http-client") - implementation("io.micronaut:micronaut-jackson-databind") - implementation("io.micronaut.groovy:micronaut-runtime-groovy") - implementation("io.micronaut.reactor:micronaut-reactor") - implementation("io.micronaut.reactor:micronaut-reactor-http-client") - implementation("jakarta.annotation:jakarta.annotation-api") - implementation("io.micronaut:micronaut-validation") + annotationProcessor 'io.micronaut.validation:micronaut-validation-processor' + annotationProcessor 'io.micronaut:micronaut-http-validation' + compileOnly 'io.micronaut.data:micronaut-data-processor' + compileOnly 'io.micronaut:micronaut-inject-groovy' + compileOnly 'io.micronaut:micronaut-http-validation' + implementation 'jakarta.persistence:jakarta.persistence-api:3.0.0' + api 'io.seqera:lib-mail:1.2.0' + api 'io.seqera:wave-api:0.14.0' + api 'io.seqera:wave-utils:0.15.0' + implementation 'io.micronaut:micronaut-http-client' + implementation 'io.micronaut:micronaut-jackson-databind' + implementation 'io.micronaut.groovy:micronaut-runtime-groovy' + implementation 'io.micronaut.reactor:micronaut-reactor' + implementation 'io.micronaut.reactor:micronaut-reactor-http-client' + implementation 'jakarta.annotation:jakarta.annotation-api' + implementation 'io.micronaut.validation:micronaut-validation' implementation 'io.micronaut.security:micronaut-security' - implementation "org.codehaus.groovy:groovy-json" - implementation "org.codehaus.groovy:groovy-nio" - implementation 'com.google.guava:guava:32.1.2-jre' + implementation 'io.micronaut:micronaut-websocket' + implementation 'org.apache.groovy:groovy-json' + implementation 'org.apache.groovy:groovy-nio' + implementation 'com.google.guava:guava:33.3.1-jre' implementation 'dev.failsafe:failsafe:3.1.0' - implementation('io.projectreactor:reactor-core') - implementation("io.seqera:tower-crypto:22.4.0-watson") { transitive = false } // to be replaced with 22.4.0 once released - implementation 'org.apache.commons:commons-compress:1.24.0' - implementation 'org.apache.commons:commons-lang3:3.12.0' + implementation 'io.micronaut.reactor:micronaut-reactor' + implementation 'io.micronaut.reactor:micronaut-reactor-http-client' + implementation('io.seqera:tower-crypto:22.4.0-watson') { transitive = false } // to be replaced with 22.4.0 once released + implementation 'org.apache.commons:commons-compress:1.27.1' + implementation 'org.apache.commons:commons-lang3:3.17.0' implementation 'io.kubernetes:client-java:19.0.0' implementation 'io.kubernetes:client-java-api-fluent:18.0.1' implementation 'com.google.code.gson:gson:2.9.0' - implementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" + implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' - implementation 'com.squareup.moshi:moshi:1.14.0' - implementation 'com.squareup.moshi:moshi-adapters:1.14.0' - implementation 'redis.clients:jedis:5.0.2' - implementation "io.github.resilience4j:resilience4j-ratelimiter:0.17.0" + implementation 'com.squareup.moshi:moshi:1.15.1' + implementation 'com.squareup.moshi:moshi-adapters:1.15.1' + implementation 'redis.clients:jedis:5.1.3' + implementation 'io.github.resilience4j:resilience4j-ratelimiter:0.17.0' + implementation 'io.micronaut:micronaut-retry' // caching deps - implementation("io.micronaut.cache:micronaut-cache-core") - implementation("io.micronaut.cache:micronaut-cache-caffeine") - implementation("io.micronaut.aws:micronaut-aws-parameter-store") - implementation "software.amazon.awssdk:ecr" - implementation "software.amazon.awssdk:ecrpublic" + implementation 'io.micronaut.cache:micronaut-cache-core' + implementation 'io.micronaut.cache:micronaut-cache-caffeine' + implementation 'io.micronaut.aws:micronaut-aws-parameter-store' + implementation 'software.amazon.awssdk:ecr' + implementation 'software.amazon.awssdk:ecrpublic' implementation 'software.amazon.awssdk:ses' - implementation 'org.yaml:snakeyaml:2.0' + implementation 'org.yaml:snakeyaml:2.2' implementation 'com.github.ben-manes.caffeine:caffeine:3.1.8' implementation 'org.luaj:luaj-jse:3.0.1' //object storage dependency - implementation("io.micronaut.objectstorage:micronaut-object-storage-aws") + implementation 'io.micronaut.objectstorage:micronaut-object-storage-aws' // include sts to allow the use of service account role - https://stackoverflow.com/a/73306570 // this sts dependency is require by micronaut-aws-parameter-store, // not directly used by the app, for this reason keeping `runtimeOnly` - runtimeOnly "software.amazon.awssdk:sts" - - runtimeOnly("io.netty:netty-tcnative-boringssl-static:2.0.0.Final") - runtimeOnly("javax.xml.bind:jaxb-api:2.3.1") - testImplementation("org.testcontainers:testcontainers") - testImplementation("org.testcontainers:mysql:1.17.3") + runtimeOnly 'software.amazon.awssdk:sts' + runtimeOnly 'io.netty:netty-tcnative-boringssl-static:2.0.0.Final' + runtimeOnly 'javax.xml.bind:jaxb-api:2.3.1' + testImplementation 'org.testcontainers:testcontainers' + testImplementation 'org.testcontainers:mysql:1.17.3' // -- - implementation("ch.qos.logback:logback-classic:1.4.8") + implementation 'ch.qos.logback:logback-classic:1.5.12' // rate limit - implementation 'com.github.seqeralabs:spillway:7b72700293' + implementation 'com.coveo:spillway:3.0.0' // monitoring - implementation "io.micronaut.micrometer:micronaut-micrometer-registry-prometheus" + implementation 'io.micronaut.micrometer:micronaut-micrometer-registry-prometheus' // Also required to enable endpoint - implementation "io.micronaut:micronaut-management" + implementation 'io.micronaut:micronaut-management' //views - implementation("io.micronaut.views:micronaut-views-handlebars") + implementation 'io.micronaut.views:micronaut-views-handlebars' + + // upgrade indirect dependencies + runtimeOnly 'org.bouncycastle:bcpkix-jdk18on:1.78' + runtimeOnly 'org.bitbucket.b_c:jose4j:0.9.4' + runtimeOnly 'io.netty:netty-bom:4.1.115.Final' } application { @@ -148,8 +156,7 @@ jib { run{ def envs = findProperty('micronautEnvs') - // note: "--enable-preview" is required to use virtual threads on Java 19 and 20 - def args = ["-Dmicronaut.environments=$envs","--enable-preview"] + def args = ["-Dmicronaut.environments=$envs","-Djdk.tracePinnedThreads=short"] if( environment['JVM_OPTS'] ) args.add(environment['JVM_OPTS']) jvmArgs args systemProperties 'DOCKER_USER': project.findProperty('DOCKER_USER') ?: environment['DOCKER_USER'], diff --git a/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle index 0cf39fc5e..4ff807f4c 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.groovy-application-conventions.gradle @@ -11,8 +11,3 @@ plugins { } group = 'io.seqera' - -tasks.withType(Test) { - // note: "--enable-preview" is required to use virtual thread on Java 19 and 20 - jvmArgs (["--enable-preview"]) -} diff --git a/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle index da3059f91..45696fae9 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.groovy-common-conventions.gradle @@ -14,17 +14,17 @@ repositories { java { toolchain { - languageVersion = JavaLanguageVersion.of(19) + languageVersion = JavaLanguageVersion.of(21) } } compileJava { - options.release.set(11) + options.release.set(17) } -tasks.withType(GroovyCompile) { - sourceCompatibility = '11' - targetCompatibility = '11' +tasks.withType(GroovyCompile).configureEach { + sourceCompatibility = '17' + targetCompatibility = '17' } group = 'io.seqera' diff --git a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle index f6197b641..835199bd4 100644 --- a/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle +++ b/buildSrc/src/main/groovy/io.seqera.wave.java-library-conventions.gradle @@ -16,17 +16,17 @@ repositories { java { toolchain { - languageVersion = JavaLanguageVersion.of(19) + languageVersion = JavaLanguageVersion.of(21) } } compileJava { - options.release.set(11) + options.release.set(17) } -tasks.withType(GroovyCompile) { - sourceCompatibility = '11' - targetCompatibility = '11' +tasks.withType(GroovyCompile).configureEach { + sourceCompatibility = '17' + targetCompatibility = '17' } test { @@ -40,22 +40,21 @@ java { } dependencies { - implementation 'org.slf4j:slf4j-api:1.7.36' + implementation 'org.slf4j:slf4j-api:2.0.16' - testImplementation 'ch.qos.logback:logback-core:1.2.11' - testImplementation 'ch.qos.logback:logback-classic:1.2.11' - testImplementation "org.codehaus.groovy:groovy:3.0.15" - testImplementation "org.codehaus.groovy:groovy-nio:3.0.15" - testImplementation ("org.codehaus.groovy:groovy-test:3.0.17") - testImplementation ("cglib:cglib-nodep:3.3.0") - testImplementation ("org.objenesis:objenesis:3.2") - testImplementation ("org.spockframework:spock-core:2.3-groovy-3.0") { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } - testImplementation ('org.spockframework:spock-junit4:2.3-groovy-3.0') { exclude group: 'org.codehaus.groovy'; exclude group: 'net.bytebuddy' } + testImplementation 'ch.qos.logback:logback-core:1.5.12' + testImplementation 'ch.qos.logback:logback-classic:1.5.12' + testImplementation 'org.apache.groovy:groovy:4.0.15' + testImplementation 'org.apache.groovy:groovy-nio:4.0.15' + testImplementation 'org.apache.groovy:groovy-test:4.0.15' + testImplementation 'org.objenesis:objenesis:3.4' + testImplementation 'net.bytebuddy:byte-buddy:1.14.17' + testImplementation 'org.spockframework:spock-core:2.3-groovy-4.0' + testImplementation 'org.spockframework:spock-junit4:2.3-groovy-4.0' } -tasks.withType(Test) { - jvmArgs ([ - '--enable-preview', +tasks.withType(Test).configureEach { + jvmArgs([ '--add-opens=java.base/java.lang=ALL-UNNAMED', '--add-opens=java.base/java.io=ALL-UNNAMED', '--add-opens=java.base/java.nio=ALL-UNNAMED', diff --git a/changelog.txt b/changelog.txt index 41b36f097..5e5b7fdad 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,4 +1,64 @@ # Wave changelog +1.15.0 - 18 Nov 2024 +- Migration to virtual threads - phase 1 (#746) [aaf0420c] +- Use runAsync instead supplyAsync [ffd0dacd] +- Remove deprecated ThreadPoolBuilder [7af3046f] +- Replace Guava cache with Caffeine (#745) [cf813e0a] +- Update project deps [f24b684d] +- Bump guava to version 33.3.1-jre [328e9ea3] +- Bump Netty version 4.1.115.Final [9ba433ce] +- Bump gradle 8.10.2 [52272fe1] + +1.14.1 - 14 Nov 2024 +- Fix creds validation endpoint (#740) [8c0f3a4c] + +1.14.0 - 10 Nov 2024 +- Fix K8s env propagation [76f0a456] +- Remove deprecated K8s methods (#734) [481298bf] +- Bump to Micronaut 4.6 (#318) [f67e8556] +- Bump Java 21 as build requirement (#519) [132f9491] +- Bump bitbucket.b_c:jose4j:0.9.4 [2e10416a] +- Bump bouncycastle:bcpkix-jdk18on:1.78 [ede22ce5] +- Bump jedis 5.1.3 (#732) [2ee0854e] +- Bump logback 1.5.12 [f5fe3fa4] +- Bump make deps runtimeclasspath [2a342b18] +- Bump snakeyaml 2.2 [6aeb3c33] +- Bump spillway 3.0.0 (#731) [1502696d] +- Bump explicit dep to websocket module [2e413ac2] +- Enables EKS Pod identity via AWS SDK 2.27.8 + +1.13.11 - 2 Nov 2024 +- Rename async methods for semantic consistency [38114d75] +- Save scan record async (#730) [3ad82a3a] +- Cap number of vulnerabilities reported in scan report to 100 (#728) [2f0d8f9f] +- Bump org.apache.commons:commons-compress:1.27.1 (#722) [adb75007] + +1.13.10 - 29 Oct 2024 +- Log slow processing stream messages [e8a6b7ee] +- Prevent scan when mode is not defined [d42bcae1] + +1.13.9 - 29 Oct 2024 +- Fix inspect view (#725) [dcf41dea] [e38e2c44] + +1.13.8 - 26 Oct 2024 +- Fix update scan status synchronously [e767c367] +- Bump scan warn colour [705141f0] +- Improve scan logging [f01e4dba] + +1.13.7 - 25 Oct 2024 +- Add ability to configure trivy environment & DBs (#720) [0f600306] + +1.13.6 - 25 Oct 2024 +- Add scan color for different vuls (#719) [ab81b6dc] + +1.13.5 - 23 Oct 2024 +- Fix Do not render inspect url on fail [d96275a1] +- Fix inspect view empty nodes (#706) [b3473b7e] +- Fix prevent scan on cached failed builds [4473fe8c] +- Use JedisPool in place of generic connection pool (#711) [cd16cfd1] +- Minor page title change [c3be9304] +- GHA to submit dependency graph to Github (#715) [09c86627] + 1.13.4 - 20 Oct 2024 - Add scan failure duration setting (#705) [372d6dec] - Change scan config log to info [f382c51a] diff --git a/docs/cli/index.mdx b/docs/cli/index.mdx index 6581f628d..45f1239e2 100644 --- a/docs/cli/index.mdx +++ b/docs/cli/index.mdx @@ -27,8 +27,8 @@ The following CLI arguments are available for Seqera Platform integration: The following environment variables are available for Seqera Platform integration: -- `TOWER_API_ENDPOINT`: A Seqera Platform auth token so that Wave can access your private registry credentials. -- `TOWER_ACCESS_TOKEN`: For Enterprise customers, the URL endpoint for your instance, such as `https://api.cloud.seqera.io`. +- `TOWER_ACCESS_TOKEN`: A Seqera Platform auth token so that Wave can access your private registry credentials. +- `TOWER_API_ENDPOINT`: For Enterprise customers, the URL endpoint for your instance, such as `https://api.cloud.seqera.io`. - `TOWER_WORKSPACE_ID`: A Seqera Platform workspace ID, such as `1234567890`, where credentials may be stored. ## Usage limits diff --git a/gradle.properties b/gradle.properties index 645585ab7..54b62b375 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,5 +16,5 @@ # along with this program. If not, see . # -micronautVersion=3.10.3 +micronautVersion=4.6.3 micronautEnvs=dev,h2,mail,aws-ses diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a4413138c..df97d72b8 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/settings.gradle b/settings.gradle index 395412606..0605b3681 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,3 +1,10 @@ +plugins { + // required to download the toolchain (jdk) from a remote repository + // https://github.com/gradle/foojay-toolchains + // https://docs.gradle.org/current/userguide/toolchains.html#sub:download_repositories + id("org.gradle.toolchains.foojay-resolver-convention") version "0.7.0" +} + rootProject.name="wave" // only for development diff --git a/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy b/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy index b76abda4e..d76ff89ee 100644 --- a/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy +++ b/src/main/groovy/io/seqera/wave/auth/BasicAuthenticationProvider.groovy @@ -19,18 +19,17 @@ package io.seqera.wave.auth import groovy.util.logging.Slf4j +import io.micronaut.core.annotation.NonNull import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpRequest -import io.micronaut.security.authentication.AuthenticationProvider +import io.micronaut.security.authentication.AuthenticationFailureReason import io.micronaut.security.authentication.AuthenticationRequest import io.micronaut.security.authentication.AuthenticationResponse +import io.micronaut.security.authentication.provider.HttpRequestAuthenticationProvider import io.seqera.wave.service.account.AccountService import io.seqera.wave.util.StringUtils import jakarta.inject.Inject import jakarta.inject.Singleton -import org.reactivestreams.Publisher -import reactor.core.publisher.Flux -import reactor.core.publisher.FluxSink /** * Basic Authentication provider * @@ -38,25 +37,22 @@ import reactor.core.publisher.FluxSink */ @Slf4j @Singleton -class BasicAuthenticationProvider implements AuthenticationProvider { +class BasicAuthenticationProvider implements HttpRequestAuthenticationProvider { @Inject private AccountService accountService @Override - Publisher authenticate(@Nullable HttpRequest httpRequest, AuthenticationRequest authRequest) { - Flux.create(emitter -> { - final user = authRequest.identity?.toString() - final pass = authRequest.secret?.toString() - if (accountService.isAuthorised(user, pass)) { - log.trace "Auth request OK - user '$user'; password: '${StringUtils.redact(pass)}'" - emitter.next(AuthenticationResponse.success((String) authRequest.identity)) - emitter.complete() - } - else { - log.trace "Auth request FAILED - user '$user'; password: '${StringUtils.redact(pass)}'" - emitter.error(AuthenticationResponse.exception()) - } - }, FluxSink.OverflowStrategy.ERROR) + AuthenticationResponse authenticate(@Nullable HttpRequest httpRequest, @NonNull AuthenticationRequest authRequest) { + final user = authRequest.identity?.toString() + final pass = authRequest.secret?.toString() + if (accountService.isAuthorised(user, pass)) { + log.trace "Auth request OK - user '$user'; password: '${StringUtils.redact(pass)}'" + return AuthenticationResponse.success(authRequest.identity) + } + else { + log.trace "Auth request FAILED - user '$user'; password: '${StringUtils.redact(pass)}'" + return AuthenticationResponse.failure(AuthenticationFailureReason.CREDENTIALS_DO_NOT_MATCH) + } } } diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy index d8ea12d26..7ae087db7 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryAuthServiceImpl.groovy @@ -21,13 +21,12 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse import java.time.Duration -import java.util.concurrent.ExecutionException +import java.util.concurrent.CompletionException import java.util.concurrent.TimeUnit -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache -import com.google.common.util.concurrent.UncheckedExecutionException +import com.github.benmanes.caffeine.cache.AsyncLoadingCache +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine import groovy.json.JsonSlurper import groovy.transform.Canonical import groovy.transform.CompileStatic @@ -101,11 +100,12 @@ class RegistryAuthServiceImpl implements RegistryAuthService { return result } - private LoadingCache cacheTokens = CacheBuilder + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cacheTokens = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterAccess(_1_HOUR.toMillis(), TimeUnit.MILLISECONDS) - .build(loader) + .buildAsync(loader) @Inject private RegistryLookupService lookupService @@ -269,9 +269,10 @@ class RegistryAuthServiceImpl implements RegistryAuthService { protected String getAuthToken(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) try { - return cacheTokens.get(key) + // FIXME https://github.com/seqeralabs/wave/issues/747 + return cacheTokens.synchronous().get(key) } - catch (UncheckedExecutionException | ExecutionException e) { + catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup // and throws the causing exception that should be `RegistryUnauthorizedAccessException` throw e.cause @@ -287,7 +288,8 @@ class RegistryAuthServiceImpl implements RegistryAuthService { */ void invalidateAuthorization(String image, RegistryAuth auth, RegistryCredentials creds) { final key = new CacheKey(image, auth, creds) - cacheTokens.invalidate(key) + // FIXME https://github.com/seqeralabs/wave/issues/747 + cacheTokens.synchronous().invalidate(key) tokenStore.remove(getStableKey(key)) } diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy index eb43f3dad..560a784a5 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryConfig.groovy @@ -47,7 +47,7 @@ class RegistryConfig { * io: [ ... ] * ] */ - private Map registries + Map registries RegistryKeys getRegistryKeys(String registryName) { final String defaultRegistry = registries.get('default')?.toString() ?: 'docker.io' diff --git a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy index 0dc641879..2b15b8efa 100644 --- a/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/auth/RegistryLookupServiceImpl.groovy @@ -20,13 +20,12 @@ package io.seqera.wave.auth import java.net.http.HttpRequest import java.net.http.HttpResponse -import java.util.concurrent.ExecutionException +import java.util.concurrent.CompletionException import java.util.concurrent.TimeUnit -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache -import com.google.common.util.concurrent.UncheckedExecutionException +import com.github.benmanes.caffeine.cache.AsyncLoadingCache +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.configuration.HttpClientConfig @@ -74,11 +73,12 @@ class RegistryLookupServiceImpl implements RegistryLookupService { } } - private LoadingCache cache = CacheBuilder + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cache = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterAccess(1, TimeUnit.HOURS) - .build(loader) + .buildAsync(loader) protected RegistryAuth lookup0(URI endpoint) { final httpClient = HttpClientFactory.followRedirectsHttpClient() @@ -117,10 +117,11 @@ class RegistryLookupServiceImpl implements RegistryLookupService { RegistryInfo lookup(String registry) { try { final endpoint = registryEndpoint(registry) - final auth = cache.get(endpoint) + // FIXME https://github.com/seqeralabs/wave/issues/747 + final auth = cache.synchronous().get(endpoint) return new RegistryInfo(registry, endpoint, auth) } - catch (UncheckedExecutionException | ExecutionException e) { + catch (CompletionException e) { // this catches the exception thrown in the cache loader lookup // and throws the causing exception that should be `RegistryUnauthorizedAccessException` throw e.cause diff --git a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy index a0f833f27..0ff20d39c 100644 --- a/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/ScanConfig.groovy @@ -21,16 +21,14 @@ package io.seqera.wave.configuration import java.nio.file.Files import java.nio.file.Path import java.time.Duration - -import io.micronaut.context.annotation.Requires -import io.micronaut.core.annotation.Nullable import javax.annotation.PostConstruct import groovy.transform.CompileStatic import groovy.transform.Memoized import groovy.util.logging.Slf4j +import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value -import io.seqera.wave.util.StringUtils +import io.micronaut.core.annotation.Nullable import jakarta.inject.Singleton /** * Container Scan service settings @@ -83,8 +81,12 @@ class ScanConfig { Duration scanIdDuration @Nullable - @Value('${wave.scan.github-token}') - String githubToken + @Value('${wave.scan.environment}') + List environment + + @Nullable + @Value('${wave.scan.vulnerability.limit:100}') + Integer vulnerabilityLimit String getScanImage() { return scanImage @@ -93,7 +95,11 @@ class ScanConfig { @Memoized Path getCacheDirectory() { final result = Path.of(buildDirectory).toAbsolutePath().resolve('.trivy-cache') - Files.createDirectories(result) + try { + Files.createDirectories(result) + } catch (IOException e) { + log.error "Unable to create scan cache directory=${result} - cause: ${e.message}" + } return result } @@ -118,8 +124,23 @@ class ScanConfig { return severity } + List> getEnvironmentAsTuples() { + if( !environment ) + return List.of() + final result = new ArrayList>() + for( String entry : environment ) { + final p=entry.indexOf('=') + final name = p!=-1 ? entry.substring(0,p) : entry + final value = p!=-1 ? entry.substring(p+1) : '' + if( !value ) + log.warn "Invalid 'wave.scan.environment' value -- offending entry: '$entry'" + result.add(new Tuple2(name,value)) + } + return result + } + @PostConstruct private void init() { - log.info("Scanner config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; retry-attempts: $retryAttempts; github-token=${StringUtils.redact(githubToken)}") + log.info("Scan config: docker image name: ${scanImage}; cache directory: ${cacheDirectory}; timeout=${timeout}; cpus: ${requestsCpu}; mem: ${requestsMemory}; severity: $severity; vulnerability-limit: $vulnerabilityLimit; retry-attempts: $retryAttempts; env=${environment}") } } diff --git a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy index 06679fa4a..f99549087 100644 --- a/src/main/groovy/io/seqera/wave/controller/BuildController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/BuildController.groovy @@ -42,7 +42,7 @@ import jakarta.inject.Inject @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class BuildController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 713ffae89..a65568c40 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -29,6 +29,7 @@ import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpRequest import io.micronaut.http.HttpResponse +import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Delete import io.micronaut.http.annotation.Error @@ -102,7 +103,7 @@ import static java.util.concurrent.CompletableFuture.completedFuture @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ContainerController { @Inject @@ -180,14 +181,14 @@ class ContainerController { @Deprecated @Post('/container-token') - @ExecuteOn(TaskExecutors.IO) - CompletableFuture> getToken(HttpRequest httpRequest, SubmitContainerTokenRequest req) { + @ExecuteOn(TaskExecutors.BLOCKING) + CompletableFuture> getToken(HttpRequest httpRequest, @Body SubmitContainerTokenRequest req) { return getContainerImpl(httpRequest, req, false) } @Post('/v1alpha2/container') - @ExecuteOn(TaskExecutors.IO) - CompletableFuture> getTokenV2(HttpRequest httpRequest, SubmitContainerTokenRequest req) { + @ExecuteOn(TaskExecutors.BLOCKING) + CompletableFuture> getTokenV2(HttpRequest httpRequest, @Body SubmitContainerTokenRequest req) { return getContainerImpl(httpRequest, req, true) } @@ -284,7 +285,7 @@ class ContainerController { protected void storeContainerRequest0(SubmitContainerTokenRequest req, ContainerRequest data, TokenData token, String target, String ip) { try { final recrd = new WaveContainerRecord(req, data, target, ip, token.expiration) - persistenceService.saveContainerRequest(recrd) + persistenceService.saveContainerRequestAsync(recrd) } catch (Throwable e) { log.error("Unable to store container request with token: ${token}", e) diff --git a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy index 88d66f9b8..6093d4dfb 100644 --- a/src/main/groovy/io/seqera/wave/controller/InspectController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/InspectController.groovy @@ -25,6 +25,7 @@ import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.micronaut.http.HttpResponse +import io.micronaut.http.annotation.Body import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Post import io.micronaut.http.annotation.QueryValue @@ -49,7 +50,7 @@ import static io.seqera.wave.util.ContainerHelper.patchPlatformEndpoint @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class InspectController { @Inject @@ -70,7 +71,7 @@ class InspectController { private String serverUrl @Post("/v1alpha1/inspect") - CompletableFuture> inspect(ContainerInspectRequest req, @Nullable @QueryValue String platform) { + CompletableFuture> inspect(@Body ContainerInspectRequest req, @Nullable @QueryValue String platform) { if( !req.containerImage ) throw new BadRequestException("Missing 'containerImage' attribute") diff --git a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy index a9ce6f89a..f8008674e 100644 --- a/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/MetricsController.groovy @@ -52,7 +52,7 @@ import static io.micronaut.http.HttpHeaders.WWW_AUTHENTICATE @Requires(property = 'wave.metrics.enabled', value = 'true') @Secured(SecurityRule.IS_AUTHENTICATED) @Controller -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class MetricsController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy index 8aaf87a51..c9c3bc8f7 100644 --- a/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/MirrorController.groovy @@ -36,7 +36,7 @@ import jakarta.inject.Inject @Slf4j @CompileStatic @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class MirrorController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy index ec6416537..263763299 100644 --- a/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/RegistryProxyController.groovy @@ -67,7 +67,7 @@ import reactor.core.publisher.Mono @Slf4j @CompileStatic @Controller("/v2") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class RegistryProxyController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy index 230a8b4a9..43fc38a5d 100644 --- a/src/main/groovy/io/seqera/wave/controller/ScanController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ScanController.groovy @@ -39,7 +39,7 @@ import jakarta.inject.Inject @CompileStatic @Requires(bean = ContainerScanService) @Controller("/") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ScanController { @Inject diff --git a/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy b/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy index 207c6e8d5..7f0a895e4 100644 --- a/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ServiceInfoController.groovy @@ -39,7 +39,7 @@ import io.seqera.wave.util.BuildInfo @Slf4j @Controller("/") @CompileStatic -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ServiceInfoController { @Value('${wave.landing.url}') diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy index 631d0bcc8..fd23fbdc6 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateController.groovy @@ -18,27 +18,23 @@ package io.seqera.wave.controller -import javax.validation.Valid - import io.micronaut.http.annotation.Controller import io.micronaut.http.annotation.Post import io.micronaut.scheduling.TaskExecutors import io.micronaut.scheduling.annotation.ExecuteOn import io.seqera.wave.auth.RegistryAuthService import jakarta.inject.Inject -import reactor.core.publisher.Mono +import jakarta.validation.Valid -@ExecuteOn(TaskExecutors.IO) @Controller("/validate-creds") +@ExecuteOn(TaskExecutors.BLOCKING) class ValidateController { @Inject RegistryAuthService loginService @Post - Mono validateCreds(@Valid ValidateRegistryCredsRequest request){ - Mono.just( - loginService.validateUser(request.registry, request.userName, request.password) - ) + Boolean validateCreds(@Valid ValidateRegistryCredsRequest request){ + loginService.validateUser(request.registry, request.userName, request.password) } } diff --git a/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy b/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy index db465ef0a..3fbe92d07 100644 --- a/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ValidateRegistryCredsRequest.groovy @@ -18,10 +18,8 @@ package io.seqera.wave.controller -import io.micronaut.core.annotation.Nullable -import javax.validation.constraints.NotBlank - import io.micronaut.core.annotation.Introspected +import jakarta.validation.constraints.NotBlank @Introspected class ValidateRegistryCredsRequest { diff --git a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy index 1fafad4b4..68328eb36 100644 --- a/src/main/groovy/io/seqera/wave/controller/ViewController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ViewController.groovy @@ -20,6 +20,7 @@ package io.seqera.wave.controller import java.util.regex.Pattern +import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value @@ -45,6 +46,7 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanEntry +import io.seqera.wave.service.scan.ScanVulnerability import io.seqera.wave.util.JacksonHelper import jakarta.inject.Inject import static io.seqera.wave.util.DataTimeUtils.formatDuration @@ -57,7 +59,7 @@ import static io.seqera.wave.util.DataTimeUtils.formatTimestamp @Slf4j @CompileStatic @Controller("/view") -@ExecuteOn(TaskExecutors.IO) +@ExecuteOn(TaskExecutors.BLOCKING) class ViewController { @Inject @@ -221,7 +223,7 @@ class ViewController { binding.scan_url = result.scanId && result.succeeded() ? "$serverUrl/view/scans/${result.scanId}" : null binding.scan_id = result.scanId // inspect uri - binding.inspect_url = "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" + binding.inspect_url = result.succeeded() ? "$serverUrl/view/inspect?image=${result.targetImage}&platform=${result.platform}" : null // configure build logs when available if( buildLogService ) { final buildLog = buildLogService.fetchLogString(result.buildId) @@ -408,7 +410,10 @@ class ViewController { } Map makeScanViewBinding(WaveScanRecord result, Map binding=new HashMap(10)) { + final color = getScanColor(result.vulnerabilities) binding.should_refresh = !result.done() + binding.scan_color_bg = color.background + binding.scan_color_fg = color.foreground binding.scan_id = result.id binding.scan_container_image = result.containerImage ?: '-' binding.scan_platform = result.platform?.toString() ?: '-' @@ -437,4 +442,24 @@ class ViewController { return binding } + @Canonical + static class Colour { + final background + final foreground + } + + protected static Colour getScanColor(List vulnerabilities){ + boolean hasMedium = vulnerabilities.stream() + .anyMatch(v -> v.severity.equals("MEDIUM")) + boolean hasHighOrCritical = vulnerabilities.stream() + .anyMatch(v -> v.severity.equals("HIGH") || v.severity.equals("CRITICAL")) + if(hasHighOrCritical){ + return new Colour('#ffe4e2', '#e00404') + } + else if(hasMedium){ + return new Colour('#fff8c5', "#000000") + } + return new Colour('#dff0d8', '#3c763d') + } + } diff --git a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy index 909cc8d5d..bd3837e87 100644 --- a/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy +++ b/src/main/groovy/io/seqera/wave/core/ContainerAugmenter.groovy @@ -280,7 +280,7 @@ class ContainerAugmenter { return result } - synchronized protected Map layerBlob(String image, ContainerLayer layer) { + protected Map layerBlob(String image, ContainerLayer layer) { log.debug "Adding layer: $layer to image: $client.registry.name/$image" // store the layer blob in the cache final String path = "$client.registry.name/v2/$image/blobs/$layer.gzipDigest" @@ -295,7 +295,6 @@ class ContainerAugmenter { protected Tuple2 updateImageManifest(String imageName, String imageManifest, String newImageConfigDigest, newImageConfigSize, boolean oci) { - // turn the json string into a json map // and append the new layer final manifest = (Map) new JsonSlurper().parseText(imageManifest) diff --git a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy index 1d9bf61de..01ed6bd94 100644 --- a/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy +++ b/src/main/groovy/io/seqera/wave/core/RegistryProxyService.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.core +import java.util.concurrent.CompletableFuture + import groovy.transform.CompileStatic import groovy.transform.ToString import groovy.util.logging.Slf4j @@ -133,7 +135,7 @@ class RegistryProxyService { return try { - persistenceService.updateContainerRequest(route.token, digest) + persistenceService.updateContainerRequestAsync(route.token, digest) } catch (Throwable t) { log.error("Unable store container request for token: $route.token", t) } @@ -193,7 +195,7 @@ class RegistryProxyService { String getImageDigest(String containerImage, PlatformId identity, boolean retryOnNotFound=false) { try { - return getImageDigest0(containerImage, identity, retryOnNotFound) + return getImageDigest0(containerImage, identity, retryOnNotFound).get() } catch(Exception e) { log.warn "Unable to retrieve digest for image '${containerImage}' -- cause: ${e.message}" @@ -203,8 +205,15 @@ class RegistryProxyService { static private List RETRY_ON_NOT_FOUND = HTTP_RETRYABLE_ERRORS + 404 + // note: return a CompletableFuture to force micronaut to use caffeine AsyncCache + // that provides a workaround about the use of virtual threads with SyncCache + // see https://github.com/ben-manes/caffeine/issues/1468#issuecomment-1906733926 @Cacheable(value = 'cache-registry-proxy', atomic = true, parameters = ['image']) - protected String getImageDigest0(String image, PlatformId identity, boolean retryOnNotFound) { + protected CompletableFuture getImageDigest0(String image, PlatformId identity, boolean retryOnNotFound) { + CompletableFuture.completedFuture(getImageDigest1(image, identity, retryOnNotFound)) + } + + protected String getImageDigest1(String image, PlatformId identity, boolean retryOnNotFound) { final coords = ContainerCoordinates.parse(image) final route = RoutePath.v2manifestPath(coords, identity) final proxyClient = client(route) diff --git a/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy b/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy index b54ac876d..fb17c68ec 100644 --- a/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy +++ b/src/main/groovy/io/seqera/wave/exchange/PairingRequest.groovy @@ -18,11 +18,10 @@ package io.seqera.wave.exchange -import javax.validation.constraints.NotBlank -import javax.validation.constraints.NotNull - import groovy.transform.CompileStatic import io.micronaut.core.annotation.Introspected +import jakarta.validation.constraints.NotBlank +import jakarta.validation.constraints.NotNull /** * Model the request for a remote service instance to register diff --git a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy index da19fc7c6..bb70548ac 100644 --- a/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy +++ b/src/main/groovy/io/seqera/wave/filter/PullMetricsRequestsFilter.groovy @@ -83,10 +83,10 @@ class PullMetricsRequestsFilter implements HttpServerFilter { final contentType = response.headers.get(HttpHeaders.CONTENT_TYPE) if( contentType && contentType in MANIFEST_TYPES ) { final route = routeHelper.parse(request.path) - CompletableFuture.supplyAsync(() -> metricsService.incrementPullsCounter(route.identity), executor) + CompletableFuture.runAsync(() -> metricsService.incrementPullsCounter(route.identity), executor) final version = route.request?.containerConfig?.fusionVersion() if (version) { - CompletableFuture.supplyAsync(() -> metricsService.incrementFusionPullsCounter(route.identity), executor) + CompletableFuture.runAsync(() -> metricsService.incrementFusionPullsCounter(route.identity), executor) } } } diff --git a/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy b/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy index f1301037e..46c9730d2 100644 --- a/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy +++ b/src/main/groovy/io/seqera/wave/http/HttpClientFactory.groovy @@ -22,6 +22,7 @@ import java.net.http.HttpClient import java.time.Duration import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.locks.ReentrantLock import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -39,9 +40,9 @@ class HttpClientFactory { static private Duration timeout = Duration.ofSeconds(20) - static private final Object l1 = new Object() + static private final ReentrantLock l1 = new ReentrantLock() - static private final Object l2 = new Object() + static private final ReentrantLock l2 = new ReentrantLock() private static HttpClient client1 @@ -51,20 +52,26 @@ class HttpClientFactory { static HttpClient followRedirectsHttpClient() { if( client1!=null ) return client1 - synchronized (l1) { + l1.lock() + try { if( client1!=null ) return client1 return client1=followRedirectsHttpClient0() + } finally { + l1.unlock() } } static HttpClient neverRedirectsHttpClient() { if( client2!=null ) return client2 - synchronized (l2) { + l2.lock() + try { if( client2!=null ) return client2 return client2=neverRedirectsHttpClient0() + } finally { + l2.unlock() } } diff --git a/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy b/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy index 7283fb205..5905f539e 100644 --- a/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy +++ b/src/main/groovy/io/seqera/wave/proxy/ErrResponse.groovy @@ -90,17 +90,17 @@ class ErrResponse implements HttpResponse { } static ErrResponse forString(String msg, HttpRequest request) { - final head = HttpHeaders.of('Content-Type': ['text/plain'], {true}) + final head = HttpHeaders.of('Content-Type': ['text/plain'], (a, b) -> true) new ErrResponse(statusCode: 400, body: msg, request: request, uri: request.uri(), headers: head) } static ErrResponse forStream(String msg, HttpRequest request) { - final head = HttpHeaders.of('Content-Type': ['text/plain'], {true}) + final head = HttpHeaders.of('Content-Type': ['text/plain'], (a, b) -> true) new ErrResponse(statusCode: 400, body: new ByteArrayInputStream(msg.bytes), request: request, uri: request.uri(), headers: head) } static ErrResponse forByteArray(String msg, HttpRequest request) { - final head = HttpHeaders.of('Content-Type': ['text/plain'], {true}) + final head = HttpHeaders.of('Content-Type': ['text/plain'], (a, b) -> true) new ErrResponse(statusCode: 400, body: msg.bytes, request: request, uri: request.uri(), headers: head) } } diff --git a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy index c7431642f..cc6f9d036 100644 --- a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy +++ b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillWayStorageFactory.groovy @@ -18,8 +18,6 @@ package io.seqera.wave.ratelimit.impl -import javax.validation.constraints.NotNull - import com.coveo.spillway.storage.InMemoryStorage import com.coveo.spillway.storage.LimitUsageStorage import com.coveo.spillway.storage.RedisStorage @@ -27,9 +25,9 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Requires -import io.seqera.wave.configuration.RateLimiterConfig import io.seqera.wave.configuration.RedisConfig import jakarta.inject.Singleton +import jakarta.validation.constraints.NotNull import redis.clients.jedis.JedisPool /** diff --git a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy index 358e951dd..e2c9eab60 100644 --- a/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy +++ b/src/main/groovy/io/seqera/wave/ratelimit/impl/SpillwayRateLimiter.groovy @@ -18,8 +18,6 @@ package io.seqera.wave.ratelimit.impl -import javax.validation.constraints.NotNull - import com.coveo.spillway.Spillway import com.coveo.spillway.SpillwayFactory import com.coveo.spillway.limit.Limit @@ -34,6 +32,7 @@ import io.seqera.wave.exception.SlowDownException import io.seqera.wave.ratelimit.AcquireRequest import io.seqera.wave.ratelimit.RateLimiterService import jakarta.inject.Singleton +import jakarta.validation.constraints.NotNull /** * This class manage how many requests can be requested from an user during a configurable period * @@ -60,7 +59,7 @@ class SpillwayRateLimiter implements RateLimiterService { init(storage, config) } - protected void init(@NotNull LimitUsageStorage storage, @NotNull RateLimiterConfig config){ + protected void init(LimitUsageStorage storage, RateLimiterConfig config){ SpillwayFactory spillwayFactory = new SpillwayFactory(storage) initBuilds(spillwayFactory, config) initPulls(spillwayFactory, config) diff --git a/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy index 3cccda9ab..f8d6bb4f1 100644 --- a/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/account/AccountServiceImpl.groovy @@ -36,7 +36,7 @@ import jakarta.annotation.PostConstruct @ConfigurationProperties('wave') class AccountServiceImpl implements AccountService { - private Map accounts = Map.of() + Map accounts = Map.of() @PostConstruct private dumpAccounts() { diff --git a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy index 17fdb3720..30f9d0e04 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/AwsEcrService.groovy @@ -21,9 +21,9 @@ package io.seqera.wave.service.aws import java.util.concurrent.TimeUnit import java.util.regex.Pattern -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache +import com.github.benmanes.caffeine.cache.AsyncLoadingCache +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.Canonical import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -73,11 +73,12 @@ class AwsEcrService { } } - private LoadingCache cache = CacheBuilder + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncLoadingCache cache = Caffeine.newBuilder() .newBuilder() .maximumSize(10_000) .expireAfterWrite(3, TimeUnit.HOURS) - .build(loader) + .buildAsync(loader) private EcrClient ecrClient(String accessKey, String secretKey, String region) { @@ -126,7 +127,8 @@ class AwsEcrService { try { // get the token from the cache, if missing the it's automatically // fetch using the AWS ECR client - return cache.get(new AwsCreds(accessKey,secretKey,region,isPublic)) + // FIXME https://github.com/seqeralabs/wave/issues/747 + return cache.synchronous().get(new AwsCreds(accessKey,secretKey,region,isPublic)) } catch (Exception e) { final type = isPublic ? "ECR public" : "ECR" diff --git a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy index 0d36875d5..895a90210 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/impl/ContainerBuildServiceImpl.groovy @@ -29,7 +29,6 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher import io.micronaut.core.annotation.Nullable -import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.BuildContext import io.seqera.wave.auth.RegistryCredentialsProvider @@ -191,8 +190,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler metricsService.incrementBuildsCounter(request.identity), executor) + CompletableFuture + .runAsync(() -> metricsService.incrementBuildsCounter(request.identity), executor) // launch the build async CompletableFuture @@ -326,16 +325,14 @@ class ContainerBuildServiceImpl implements ContainerBuildService, JobHandler implements Runnable { final private String name0 - final private Cache closedClients = CacheBuilder + // FIXME https://github.com/seqeralabs/wave/issues/747 + final private AsyncCache closedClients = Caffeine.newBuilder() .newBuilder() .expireAfterWrite(10, TimeUnit.MINUTES) - .build() + .buildAsync() AbstractMessageQueue(MessageQueue broker) { final type = TypeHelper.getGenericType(this, 0) @@ -149,13 +150,15 @@ abstract class AbstractMessageQueue implements Runnable { @Override void run() { + // FIXME https://github.com/seqeralabs/wave/issues/747 + final clientsCache = closedClients.synchronous() while( !thread.isInterrupted() ) { try { int sent=0 final clients = new HashMap>(this.clients) for( Map.Entry> entry : clients ) { // ignore clients marked as closed - if( closedClients.getIfPresent(entry.key)) + if( clientsCache.getIfPresent(entry.key)) continue // infer the target queue from the client key final target = targetFromClientKey(entry.key) @@ -173,7 +176,7 @@ abstract class AbstractMessageQueue implements Runnable { // offer back the value to be processed again broker.offer(target, value) if( e.message?.contains('close') ) { - closedClients.put(entry.key, true) + clientsCache.put(entry.key, true) } } } diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy index 560a81306..ae16d7140 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/AbstractMessageStream.groovy @@ -98,6 +98,10 @@ abstract class AbstractMessageStream implements Closeable { * The {@link Predicate} to be invoked when a stream message is consumed (read from) the stream. */ void addConsumer(String streamId, MessageConsumer consumer) { + // the use of synchronized block is meant to prevent a race condition while + // updating the 'listeners' from concurrent invocations. + // however, considering the addConsumer is invoked during the initialization phase + // (and therefore in the same thread) in should not be really needed. synchronized (listeners) { if( listeners.containsKey(streamId)) throw new IllegalStateException("Only one consumer can be defined for each stream - offending streamId=$streamId; consumer=$consumer") diff --git a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy index 609b71061..c417568b3 100644 --- a/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy +++ b/src/main/groovy/io/seqera/wave/service/data/stream/impl/RedisMessageStream.groovy @@ -62,6 +62,9 @@ class RedisMessageStream implements MessageStream { @Value('${wave.message-stream.claim-timeout:5s}') private Duration claimTimeout + @Value('${wave.message-stream.consume-warn-timeout-millis:4000}') + private long consumeWarnTimeoutMillis + private String consumerName @PostConstruct @@ -102,11 +105,17 @@ class RedisMessageStream implements MessageStream { @Override boolean consume(String streamId, MessageConsumer consumer) { try (Jedis jedis = pool.getResource()) { + String msg + final long begin = System.currentTimeMillis() final entry = claimMessage(jedis,streamId) ?: readMessage(jedis, streamId) - if( entry && consumer.accept(entry.getFields().get(DATA_FIELD)) ) { + if( entry && consumer.accept(msg=entry.getFields().get(DATA_FIELD)) ) { final tx = jedis.multi() // acknowledge the entry has been processed so that it cannot be claimed anymore tx.xack(streamId, CONSUMER_GROUP_NAME, entry.getID()) + final delta = System.currentTimeMillis()-begin + if( delta>consumeWarnTimeoutMillis ) { + log.warn "Redis message stream - consume processing took ${Duration.ofMillis(delta)} - offending entry=${entry.getID()}; message=${msg}" + } // this remove permanently the entry from the stream tx.xdel(streamId, entry.getID()) tx.exec() diff --git a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy index 1c4ec6c03..e10cde8d4 100644 --- a/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy +++ b/src/main/groovy/io/seqera/wave/service/job/JobManager.groovy @@ -21,6 +21,7 @@ package io.seqera.wave.service.job import java.time.Duration import java.time.Instant +import com.github.benmanes.caffeine.cache.AsyncCache import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileStatic @@ -50,16 +51,19 @@ class JobManager { @Inject private JobConfig config - private Cache debounceCache + // FIXME https://github.com/seqeralabs/wave/issues/747 + private AsyncCache debounceCache @PostConstruct void init() { log.info "Creating job manager - config=$config" - debounceCache = Caffeine.newBuilder().expireAfterWrite(config.graceInterval.multipliedBy(2)).build() + debounceCache = Caffeine + .newBuilder() + .expireAfterWrite(config.graceInterval.multipliedBy(2)) + .buildAsync() queue.addConsumer((job)-> processJob(job)) } - protected boolean processJob(JobSpec jobSpec) { try { return processJob0(jobSpec) @@ -73,7 +77,8 @@ class JobManager { } protected JobState state(JobSpec job) { - return state0(job, config.graceInterval, debounceCache) + // FIXME https://github.com/seqeralabs/wave/issues/747 + return state0(job, config.graceInterval, debounceCache.synchronous()) } protected JobState state0(final JobSpec job, final Duration graceInterval, final Cache cache) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 0d8de9015..ccaedec07 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -23,11 +23,9 @@ import java.time.Duration import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.MirrorConfig - +import io.seqera.wave.configuration.ScanConfig /** * Defines Kubernetes operations * @@ -43,23 +41,6 @@ interface K8sService { void deletePod(String name) - @Deprecated - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) - - @Deprecated - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) - - @Deprecated - Integer waitPodCompletion(V1Pod pod, long timeout) - - @Deprecated - void deletePodWhenReachStatus(String podName, String statusName, long timeout) - - @Deprecated - V1Job createJob(String name, String containerImage, List args) - - V1Job getJob(String name) - JobStatus getJobStatus(String name) void deleteJob(String name) @@ -72,9 +53,6 @@ interface K8sService { V1Job launchMirrorJob(String name, String containerImage, List args, Path workDir, Path creds, MirrorConfig config) - @Deprecated - V1PodList waitJob(V1Job job, Long timeout) - V1Pod getLatestPodForJob(String jobName) } diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index ac46c912d..c954794c2 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -22,7 +22,6 @@ import java.nio.file.Path import java.time.Duration import javax.annotation.PostConstruct -import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.kubernetes.client.custom.Quantity @@ -36,7 +35,6 @@ import io.kubernetes.client.openapi.models.V1JobStatus import io.kubernetes.client.openapi.models.V1PersistentVolumeClaimVolumeSource import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodBuilder -import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1ResourceRequirements import io.kubernetes.client.openapi.models.V1Volume import io.kubernetes.client.openapi.models.V1VolumeMount @@ -46,9 +44,9 @@ import io.micronaut.context.annotation.Value import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig +import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform -import io.seqera.wave.configuration.MirrorConfig import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton @@ -136,61 +134,6 @@ class K8sServiceImpl implements K8sService { } } - /** - * Create a K8s job with the specified name - * - * @param name - * The K8s job name. It must be unique - * @param containerImage - * The container image to be used to run the job - * @param args - * The command to be executed by the job - * @return - * An instance of {@link V1Job} - */ - @Override - @CompileDynamic - @Deprecated - V1Job createJob(String name, String containerImage, List args) { - - V1Job body = new V1JobBuilder() - .withNewMetadata() - .withNamespace(namespace) - .withName(name) - .endMetadata() - .withNewSpec() - .withBackoffLimit(0) - .withNewTemplate() - .editOrNewSpec() - .addNewContainer() - .withName(name) - .withImage(containerImage) - .withArgs(args) - .endContainer() - .withRestartPolicy("Never") - .endSpec() - .endTemplate() - .endSpec() - .build() - - return k8sClient - .batchV1Api() - .createNamespacedJob(namespace, body, null, null, null,null) - } - - /** - * Get a Jobs Job. - * - * @param name The job name - * @return An instance of {@link V1Job} - */ - @Override - V1Job getJob(String name) { - k8sClient - .batchV1Api() - .readNamespacedJob(name, namespace, null) - } - /** * Get a Job status * @@ -307,31 +250,7 @@ class K8sServiceImpl implements K8sService { .subPath(rel) } - /** - * Create a container for container image building via buildkit - * - * @param name - * The name of pod - * @param containerImage - * The container image to be used - * @param args - * The build command to be performed - * @param workDir - * The build context directory - * @param creds - * The target container repository credentials - * @return - * The {@link V1Pod} description the submitted pod - */ - @Override @Deprecated - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector) - return k8sClient - .coreV1Api() - .createNamespacedPod(namespace, spec, null, null, null,null) - } - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, Map nodeSelector) { // dirty dependency to avoid introducing another parameter @@ -408,47 +327,6 @@ class K8sServiceImpl implements K8sService { builder.build() } - /** - * Wait for a pod a completion. - * - * NOTE: this method assumes the pod is running exactly *one* container. - * - * @param pod - * The pod name - * @param timeout - * Max wait time in milliseconds - * @return - * An Integer value representing the container exit code or {@code null} if the state cannot be determined - * or timeout was reached. - */ - @Override - @Deprecated - Integer waitPodCompletion(V1Pod pod, long timeout) { - final start = System.currentTimeMillis() - // wait for termination - while( true ) { - final phase = pod.status?.phase - if( phase && phase != 'Pending' ) { - final status = pod.status.containerStatuses.first() - if( !status ) - return null - if( !status.state ) - return null - if( status.state.terminated ) { - return status.state.terminated.exitCode - } - } - - if( phase == 'Failed' ) - return null - final delta = System.currentTimeMillis()-start - if( delta > timeout ) - return null - sleep 5_000 - pod = getPod(pod.metadata.name) - } - } - /** * Fetch the logs of a pod. * @@ -481,36 +359,6 @@ class K8sServiceImpl implements K8sService { .deleteNamespacedPod(name, namespace, (String)null, (String)null, (Integer)null, (Boolean)null, (String)null, (V1DeleteOptions)null) } - /** - * Delete a pod where the status is reached - * - * @param name The name of the pod to be deleted - * @param statusName The status to be reached - * @param timeout The max wait time in milliseconds - */ - @Override - @Deprecated - void deletePodWhenReachStatus(String podName, String statusName, long timeout){ - final pod = getPod(podName) - final start = System.currentTimeMillis() - while( (System.currentTimeMillis() - start) < timeout ) { - if( pod?.status?.phase == statusName ) { - deletePod(podName) - return - } - sleep 5_000 - } - } - - @Override - @Deprecated - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { - final spec = scanSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) - return k8sClient - .coreV1Api() - .createNamespacedPod(namespace, spec, null, null, null,null) - } - @Deprecated V1Pod scanSpec(String name, String containerImage, List args, Path workDir, Path credsFile, ScanConfig scanConfig, Map nodeSelector) { @@ -784,8 +632,11 @@ class K8sServiceImpl implements K8sService { .withVolumeMounts(mounts) .withResources(requests) - if( scanConfig.githubToken ) { - container.withEnv(new V1EnvVar().name('GITHUB_TOKEN').value(scanConfig.githubToken)) + final env = scanConfig.environmentAsTuples + for( Tuple2 entry : env ) { + final String k = entry.v1 + final String v = entry.v2 + container.addToEnv(new V1EnvVar().name(k).value(v)) } // spec section @@ -862,33 +713,6 @@ class K8sServiceImpl implements K8sService { return result } - /** - * Wait for a job to complete - * - * @param k8s job - * @param timeout - * Max wait time in milliseconds - * @return list of pods created by the job - */ - @Deprecated - @Override - V1PodList waitJob(V1Job job, Long timeout) { - sleep 5_000 - final startTime = System.currentTimeMillis() - // wait for termination - while (System.currentTimeMillis() - startTime < timeout) { - final name = job.metadata.name - final status = getJobStatus(name) - if (status != JobStatus.Pending) { - return k8sClient - .coreV1Api() - .listNamespacedPod(namespace, null, null, null, null, "job-name=$name", null, null, null, null, null, null) - } - job = getJob(name) - } - return null - } - /** * Delete a job * diff --git a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy index 8d061a86e..1126fe87d 100644 --- a/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceImpl.groovy @@ -75,7 +75,7 @@ class ContainerMirrorServiceImpl implements ContainerMirrorService, JobHandler metricsService.incrementMirrorsCounter(request.identity), ioExecutor) + CompletableFuture.runAsync(() -> metricsService.incrementMirrorsCounter(request.identity), ioExecutor) jobService.launchMirror(request) return new BuildTrack(request.mirrorId, request.targetImage, false, null) } @@ -124,9 +124,14 @@ class ContainerMirrorServiceImpl implements ContainerMirrorService, JobHandler mirrorStore = new HashMap<>() @Override - void saveBuild(WaveBuildRecord record) { + void saveBuildAsync(WaveBuildRecord record) { buildStore[record.buildId] = record } @@ -76,12 +76,12 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveContainerRequest(WaveContainerRecord data) { + void saveContainerRequestAsync(WaveContainerRecord data) { requestStore.put(data.id, data) } @Override - void updateContainerRequest(String token, ContainerDigestPair digest) { + void updateContainerRequestAsync(String token, ContainerDigestPair digest) { final data = requestStore.get(token) if( data ) { requestStore.put(token, new WaveContainerRecord(data, digest.source, digest.target)) @@ -99,7 +99,7 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveScanRecord(WaveScanRecord scanRecord) { + void saveScanRecordAsync(WaveScanRecord scanRecord) { scanStore.put(scanRecord.id, scanRecord) } @@ -129,7 +129,7 @@ class LocalPersistenceService implements PersistenceService { } @Override - void saveMirrorResult(MirrorResult mirror) { + void saveMirrorResultAsync(MirrorResult mirror) { mirrorStore.put(mirror.mirrorId, mirror) } diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy index badd7c4aa..2901ec22f 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealClient.groovy @@ -54,9 +54,15 @@ interface SurrealClient { @Post("/sql") Flux> sqlAsync(@Header String authorization, @Body String body) + @Post("/sql") + Flux>> sqlAsyncMany(@Header String authorization, @Body String body) + @Post("/sql") Map sqlAsMap(@Header String authorization, @Body String body) + @Post("/sql") + List> sqlAsList(@Header String authorization, @Body String body) + @Post("/sql") String sqlAsString(@Header String authorization, @Body String body) diff --git a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy index 74079f1b7..829c6a384 100644 --- a/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy +++ b/src/main/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceService.groovy @@ -100,7 +100,7 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveBuild(WaveBuildRecord build) { + void saveBuildAsync(WaveBuildRecord build) { // note: use surreal sql in order to by-pass issue with large payload // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 final query = "INSERT INTO wave_build ${JacksonHelper.toJson(build)}" @@ -198,7 +198,7 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveContainerRequest(WaveContainerRecord data) { + void saveContainerRequestAsync(WaveContainerRecord data) { // note: use surreal sql in order to by-pass issue with large payload // see https://github.com/seqeralabs/wave/issues/559#issuecomment-2369412170 final query = "INSERT INTO wave_request ${JacksonHelper.toJson(data)}" @@ -216,7 +216,7 @@ class SurrealPersistenceService implements PersistenceService { }) } - void updateContainerRequest(String token, ContainerDigestPair digest) { + void updateContainerRequestAsync(String token, ContainerDigestPair digest) { final query = """\ UPDATE wave_request:$token SET sourceDigest = '$digest.source', @@ -253,28 +253,39 @@ class SurrealPersistenceService implements PersistenceService { } @Override - void saveScanRecord(WaveScanRecord scanRecord) { + void saveScanRecordAsync(WaveScanRecord scanRecord) { final vulnerabilities = scanRecord.vulnerabilities ?: List.of() + // create a multi-command surreal sql statement to insert all vulnerabilities + // and the scan record in a single operation + List ids = new ArrayList<>(101) + String statement = '' // save all vulnerabilities for( ScanVulnerability it : vulnerabilities ) { - surrealDb.insertScanVulnerability(authorization, it) + statement += "INSERT INTO wave_scan_vuln ${JacksonHelper.toJson(it)};\n" + ids << "wave_scan_vuln:⟨$it.id⟩".toString() } - // compose the list of ids - final ids = vulnerabilities - .collect(it-> "wave_scan_vuln:⟨$it.id⟩".toString()) - - // scan object final copy = scanRecord.clone() copy.vulnerabilities = List.of() final json = JacksonHelper.toJson(copy) - // create the scan record - final statement = "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)}".toString() - final result = surrealDb.sqlAsMap(authorization, statement) - log.trace "Scan update result=$result" + // add the wave_scan record + statement += "INSERT INTO wave_scan ${patchScanVulnerabilities(json, ids)};\n".toString() + // store the statement using an async operation + surrealDb + .sqlAsyncMany(getAuthorization(), statement) + .subscribe({result -> + log.trace "Scan record save result=$result" + }, + {error-> + def msg = error.message + if( error instanceof HttpClientResponseException ){ + msg += ":\n $error.response.body" + } + log.error("Error saving scan record => ${msg}\n", error) + }) } protected String patchScanVulnerabilities(String json, List ids) { @@ -369,7 +380,7 @@ class SurrealPersistenceService implements PersistenceService { * @param mirror {@link MirrorEntry} object */ @Override - void saveMirrorResult(MirrorResult mirror) { + void saveMirrorResultAsync(MirrorResult mirror) { surrealDb.insertMirrorAsync(getAuthorization(), mirror).subscribe({ result-> log.trace "Mirror request with id '$mirror.mirrorId' saved record: ${result}" }, {error-> diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy index 044bbcb0e..19d4635a4 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanService.groovy @@ -19,11 +19,10 @@ package io.seqera.wave.service.scan import io.seqera.wave.api.ScanMode -import io.seqera.wave.service.builder.BuildEvent +import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.mirror.MirrorEntry import io.seqera.wave.service.persistence.WaveScanRecord import io.seqera.wave.service.request.ContainerRequest - /** * Declare operations to scan containers * @@ -36,7 +35,7 @@ interface ContainerScanService { void scan(ScanRequest request) - void scanOnBuild(BuildEvent build) + void scanOnBuild(BuildEntry build) void scanOnMirror(MirrorEntry entry) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index 62b089c3b..d0e3d7511 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -29,7 +29,7 @@ import io.micronaut.context.annotation.Requires import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.service.builder.BuildEvent +import io.seqera.wave.service.builder.BuildEntry import io.seqera.wave.service.builder.BuildRequest import io.seqera.wave.service.cleanup.CleanupService import io.seqera.wave.service.inspect.ContainerInspectService @@ -106,14 +106,14 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler launch(request), executor) + try { + // create a record to mark the beginning + final scan = ScanEntry.create(request) + if( scanStore.putIfAbsent(scan.scanId, scan) ) { + //start scanning of build container + CompletableFuture.runAsync(() -> launch(request), executor) + } + } + catch (Throwable e){ + log.warn "Unable to save scan result - id=${request.scanId}; cause=${e.message}", e + storeScanEntry(ScanEntry.failure(request)) + } } @Override @@ -181,14 +190,8 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler metricsService.incrementScansCounter(request.identity), executor) - // launch container scan - jobService.launchScan(request) - } + incrScanMetrics(request) + jobService.launchScan(request) } catch (Throwable e){ log.warn "Unable to save scan result - id=${request.scanId}; cause=${e.message}", e @@ -196,6 +199,16 @@ class ContainerScanServiceImpl implements ContainerScanService, JobHandler dockerWrapper(String jobName, Path scanDir, Path credsFile, String githubToken) { + protected List dockerWrapper(String jobName, Path scanDir, Path credsFile, List env) { final wrapper = ['docker','run'] wrapper.add('--detach') @@ -112,9 +112,11 @@ class DockerScanStrategy extends ScanStrategy { wrapper.add("${credsFile}:${Trivy.CONFIG_MOUNT_PATH}:ro".toString()) } - if( githubToken ) { - wrapper.add('-e') - wrapper.add("GITHUB_TOKEN="+githubToken) + if( env ) { + for( String it : env ) { + wrapper.add('-e') + wrapper.add(it) + } } return wrapper diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index 136a57a19..34baf87a2 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -65,7 +65,7 @@ class KubeScanStrategy extends ScanStrategy { @Override void scanContainer(String jobName, ScanRequest req) { - log.info("Launching container scan for request: ${req.requestId} with scanId ${req.scanId}") + log.info("Launching container scan job: $jobName for request: ${req}") try{ // create the scan dir try { diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index 9b6a9291e..33cad3b12 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -23,6 +23,7 @@ import java.time.Instant import groovy.transform.Canonical import groovy.transform.CompileStatic +import groovy.transform.ToString import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.tower.PlatformId @@ -32,6 +33,7 @@ import io.seqera.wave.tower.PlatformId * @author Paolo Di Tommaso */ @Canonical +@ToString(includeNames = true, includePackage = false) @CompileStatic class ScanRequest { diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy index 3dff74a96..bd73f9e1d 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanVulnerability.groovy @@ -37,7 +37,7 @@ import org.jetbrains.annotations.NotNull @CompileStatic class ScanVulnerability implements Comparable { - static final Map ORDER = Map.of( + static final private Map ORDER = Map.of( 'LOW', 0, 'MEDIUM', 1, 'HIGH', 2, diff --git a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy index a1055037b..a704d467a 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/TrivyResultProcessor.groovy @@ -21,6 +21,8 @@ package io.seqera.wave.service.scan import java.nio.file.Path import groovy.json.JsonSlurper +import groovy.transform.CompileDynamic +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.seqera.wave.exception.ScanRuntimeException /** @@ -30,16 +32,31 @@ import io.seqera.wave.exception.ScanRuntimeException * @author Paolo Di Tommaso */ @Slf4j +@CompileStatic class TrivyResultProcessor { - static List process(Path reportFile) { - process(reportFile.getText()) + /** + * Parse a Trivy vulnerabilities JSON file and return a list of {@link ScanVulnerability} entries + * + * @param scanFile + * The {@link Path} of the Trivy JSON file to be scanned + * @param maxEntries + * The max number of vulnerabilities that should be returned giving precedence to the + * most severe vulnerabilities e.g. one critical and one medium issues are found and + * 1 is specified as {@code maxEntries} only the critical issues is returned. + * @return + * The list of {@link ScanVulnerability} entries as parsed in from the JSON file. + */ + static List parseFile(Path scanFile, Integer maxEntries=null) { + final result = parseJson(scanFile.getText()) + return maxEntries>0 ? filter(result, maxEntries) : result } - static List process(String trivyResult) { + @CompileDynamic + static List parseJson(String scanJson) { final slurper = new JsonSlurper() try{ - final jsonMap = slurper.parseText(trivyResult) as Map + final jsonMap = slurper.parseText(scanJson) as Map return jsonMap.Results.collect { result -> result.Vulnerabilities.collect { vulnerability -> new ScanVulnerability( @@ -57,4 +74,8 @@ class TrivyResultProcessor { throw new ScanRuntimeException("Failed to parse the trivy result", e) } } + + static protected List filter(List vulnerabilities, int limit){ + vulnerabilities.toSorted((v,w) -> w.compareTo(v)).take(limit) + } } diff --git a/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy index 0bc3a0c80..e87a51e4c 100644 --- a/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/validation/ValidationServiceImpl.groovy @@ -55,7 +55,7 @@ class ValidationServiceImpl implements ValidationService { new URI(endpoint) } catch (URISyntaxException e) { - return "Invalid endpoint '${endpoint}' — cause: ${e.message}" + return "Invalid endpoint '${endpoint}' — cause: ${e.getMessage()}" } return null diff --git a/src/main/groovy/io/seqera/wave/tower/User.groovy b/src/main/groovy/io/seqera/wave/tower/User.groovy index c8dbb71d8..bce993103 100644 --- a/src/main/groovy/io/seqera/wave/tower/User.groovy +++ b/src/main/groovy/io/seqera/wave/tower/User.groovy @@ -18,12 +18,12 @@ package io.seqera.wave.tower -import javax.validation.constraints.NotNull -import javax.validation.constraints.Size - import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.ToString +import jakarta.validation.constraints.NotNull +import jakarta.validation.constraints.Size + /** * Model a tower user * diff --git a/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy b/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy index ec213628a..27e12d87f 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/connector/TowerConnector.groovy @@ -26,10 +26,10 @@ import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.function.Function -import com.google.common.cache.Cache -import com.google.common.cache.CacheBuilder -import com.google.common.cache.CacheLoader -import com.google.common.cache.LoadingCache +import com.github.benmanes.caffeine.cache.AsyncLoadingCache +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.CacheLoader +import com.github.benmanes.caffeine.cache.Caffeine import groovy.transform.CompileDynamic import groovy.transform.CompileStatic import groovy.util.logging.Slf4j @@ -82,7 +82,7 @@ abstract class TowerConnector { private SpillwayRateLimiter limiter @Inject - @Named(TaskExecutors.IO) + @Named(TaskExecutors.BLOCKING) private volatile ExecutorService ioExecutor private CacheLoader> loader = new CacheLoader>() { @@ -92,14 +92,18 @@ abstract class TowerConnector { } } - private LoadingCache> refreshCache = CacheBuilder> + private AsyncLoadingCache> refreshCache = Caffeine .newBuilder() .expireAfterWrite(1, TimeUnit.MINUTES) - .build(loader) + .buildAsync(loader) /** Only for testing - do not use */ Cache> refreshCache0() { - return refreshCache + return refreshCache.synchronous() + } + + protected ExecutorService getIoExecutor() { + return ioExecutor } /** @@ -242,7 +246,7 @@ abstract class TowerConnector { * @return The refreshed {@link JwtAuth} object */ protected CompletableFuture refreshJwtToken(String endpoint, JwtAuth auth) { - return refreshCache.get(new JwtRefreshParams(endpoint,auth)) + return refreshCache.synchronous().get(new JwtRefreshParams(endpoint,auth)) } protected CompletableFuture refreshJwtToken0(String endpoint, JwtAuth auth) { diff --git a/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy b/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy index c52646638..04c591a09 100644 --- a/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy +++ b/src/main/groovy/io/seqera/wave/tower/client/connector/WebSocketTowerConnector.groovy @@ -19,19 +19,16 @@ package io.seqera.wave.tower.client.connector import java.util.concurrent.CompletableFuture -import java.util.concurrent.ExecutorService import java.util.function.Function import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires -import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.service.pairing.socket.PairingChannel import io.seqera.wave.service.pairing.socket.msg.PairingMessage import io.seqera.wave.service.pairing.socket.msg.ProxyHttpRequest import io.seqera.wave.service.pairing.socket.msg.ProxyHttpResponse import jakarta.inject.Inject -import jakarta.inject.Named import jakarta.inject.Singleton import static io.seqera.wave.service.pairing.PairingService.TOWER_SERVICE /** @@ -49,15 +46,11 @@ class WebSocketTowerConnector extends TowerConnector { @Inject private PairingChannel channel - @Inject - @Named(TaskExecutors.IO) - private ExecutorService ioExecutor - @Override CompletableFuture sendAsync(String endpoint, ProxyHttpRequest request) { return channel .sendRequest(TOWER_SERVICE, endpoint, request) - .thenApplyAsync(Function.identity() as Function, ioExecutor) + .thenApplyAsync(Function.identity() as Function, getIoExecutor()) } } diff --git a/src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy b/src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy deleted file mode 100644 index 7afdf0e06..000000000 --- a/src/main/groovy/io/seqera/wave/util/ThreadPoolBuilder.groovy +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Wave, containers provisioning service - * Copyright (c) 2023-2024, Seqera Labs - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package io.seqera.wave.util - - - -import java.util.concurrent.BlockingQueue -import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.RejectedExecutionHandler -import java.util.concurrent.ThreadFactory -import java.util.concurrent.ThreadPoolExecutor -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger - -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -/** - * Builder class to create instance of {@link ThreadPoolExecutor} - * - * @author Paolo Di Tommaso - */ -@Slf4j -@CompileStatic -@Deprecated -class ThreadPoolBuilder { - - static AtomicInteger poolCount = new AtomicInteger() - - private String name - - private int minSize - - private int maxSize - - private BlockingQueue workQueue - - private int queueSize = -1 - - private Long keepAliveTime - - private RejectedExecutionHandler rejectionPolicy - - private ThreadFactory threadFactory - - private boolean allowCoreThreadTimeout - - String getName() { name } - - int getMinSize() { minSize } - - int getMaxSize() { maxSize } - - int getQueueSize() { queueSize } - - BlockingQueue getWorkQueue() { workQueue } - - Long getKeepAliveTime() { keepAliveTime } - - RejectedExecutionHandler getRejectionPolicy() { rejectionPolicy } - - ThreadFactory getThreadFactory() { threadFactory } - - boolean getAllowCoreThreadTimeout() { allowCoreThreadTimeout } - - ThreadPoolBuilder withName(String name) { - if( name ) { - this.name = name - this.threadFactory = new CustomThreadFactory(name) - } - return this - } - - ThreadPoolBuilder withThreadFactory(ThreadFactory threadFactory) { - assert !name || !threadFactory, "Property 'threadFactory' or 'name' was already set" - this.threadFactory = threadFactory - return this - } - - ThreadPoolBuilder withRejectionPolicy(RejectedExecutionHandler rejectionPolicy) { - this.rejectionPolicy = rejectionPolicy - return this - } - - ThreadPoolBuilder withMinSize(int min) { - this.minSize = min - return this - } - - ThreadPoolBuilder withMaxSize(int max) { - this.maxSize = max - return this - } - - ThreadPoolBuilder withQueueSize(int size) { - this.queueSize = size - this.workQueue = new LinkedBlockingQueue(size) - return this - } - - ThreadPoolBuilder withQueue(BlockingQueue workQueue) { - this.workQueue = workQueue - return this - } - - ThreadPoolBuilder withKeepAliveTime( long millis ) { - keepAliveTime = millis - return this - } - - ThreadPoolBuilder withAllowCoreThreadTimeout(boolean flag) { - this.allowCoreThreadTimeout = flag - return this - } - - ThreadPoolExecutor build() { - assert minSize <= maxSize - - if( !name ) - name = "nf-thread-pool-${poolCount.getAndIncrement()}" - - if(keepAliveTime==null) - keepAliveTime = 60_000 - if( workQueue==null ) - workQueue = new LinkedBlockingQueue<>() - if( rejectionPolicy==null ) - rejectionPolicy = new ThreadPoolExecutor.CallerRunsPolicy() - if( threadFactory==null ) - threadFactory = new CustomThreadFactory(name) - - log.debug "Creating thread pool '$name' minSize=$minSize; maxSize=$maxSize; workQueue=${workQueue.getClass().getSimpleName()}[${queueSize}]; allowCoreThreadTimeout=$allowCoreThreadTimeout" - - final result = new ThreadPoolExecutor( - minSize, - maxSize, - keepAliveTime, TimeUnit.MILLISECONDS, - workQueue, - threadFactory, - rejectionPolicy) - - result.allowCoreThreadTimeOut(allowCoreThreadTimeout) - - return result - } - - - static ThreadPoolExecutor io(String name=null) { - io(10, 100, 10_000, name) - } - - - static ThreadPoolExecutor io(int min, int max, int queue, String name=null) { - new ThreadPoolBuilder() - .withMinSize(min) - .withMaxSize(max) - .withQueueSize(queue) - .withName(name) - .build() - } - -} diff --git a/src/main/resources/application-dev.yml b/src/main/resources/application-dev.yml index 7f15e570a..4481f847d 100644 --- a/src/main/resources/application-dev.yml +++ b/src/main/resources/application-dev.yml @@ -4,7 +4,6 @@ wave: enabled: true failure: duration: '30s' - github-token: "${GITHUB_TOKEN:}" build: workspace: 'build-workspace' metrics: diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 38d685608..bd030f589 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -84,7 +84,11 @@ wave: multiplier: '1.75' scan: image: - name: aquasec/trivy:0.55.0 + name: aquasec/trivy:0.56.2 + environment: + # see https://github.com/aquasecurity/trivy/discussions/7668#discussioncomment-11028887 + - "TRIVY_DB_REPOSITORY=public.ecr.aws/aquasecurity/trivy-db" + - "TRIVY_JAVA_DB_REPOSITORY=public.ecr.aws/aquasecurity/trivy-java-db" blobCache: s5cmdImage: public.cr.seqera.io/wave/s5cmd:v2.2.2 --- diff --git a/src/main/resources/io/seqera/wave/build-view.hbs b/src/main/resources/io/seqera/wave/build-view.hbs index 65bb3c9cd..509546fc0 100644 --- a/src/main/resources/io/seqera/wave/build-view.hbs +++ b/src/main/resources/io/seqera/wave/build-view.hbs @@ -125,8 +125,10 @@ Container image {{#if build_in_progress}} {{build_image}} - {{else}} + {{else if inspect_url}} {{build_image}} + {{else}} + {{build_image}} {{/if}} diff --git a/src/main/resources/io/seqera/wave/inspect-view.hbs b/src/main/resources/io/seqera/wave/inspect-view.hbs index 03665443e..030977efe 100644 --- a/src/main/resources/io/seqera/wave/inspect-view.hbs +++ b/src/main/resources/io/seqera/wave/inspect-view.hbs @@ -127,7 +127,6 @@ {{else}} - {{/if}}

Container Specification

@@ -210,7 +209,7 @@ details.appendChild(nestedUl); li.appendChild(details); } else { - li.textContent = `${key}: ${data[key]}`; + li.textContent = `${key}: ${JSON.stringify(data[key])}`; } ul.appendChild(li); @@ -223,5 +222,6 @@ document.getElementById('config-div').appendChild(createTreeView({{{config}}})); document.getElementById('manifest-div').appendChild(createTreeView({{{manifest}}})); +{{/if}} diff --git a/src/main/resources/io/seqera/wave/scan-view.hbs b/src/main/resources/io/seqera/wave/scan-view.hbs index 0ea0ecc1b..66ca925c8 100644 --- a/src/main/resources/io/seqera/wave/scan-view.hbs +++ b/src/main/resources/io/seqera/wave/scan-view.hbs @@ -30,13 +30,13 @@ body { {{#if scan_exist}} {{#if scan_completed}} {{#if scan_failed}} -
+

Unable to complete the container security scan successfully

{{else}} -
+

Container security scan completed

diff --git a/src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy b/src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy new file mode 100644 index 000000000..c1447b452 --- /dev/null +++ b/src/test/groovy/io/seqera/wave/configuration/ScanConfigTest.groovy @@ -0,0 +1,41 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.configuration + +import spock.lang.Specification + +/** + * + * @author Paolo Di Tommaso + */ +class ScanConfigTest extends Specification { + + def 'should convert env to tuples' () { + given: + def config1 = new ScanConfig() + def config2 = new ScanConfig(environment: ['FOO=one','BAR=two']) + def config3 = new ScanConfig(environment: ['FOO','BAR=']) + + expect: + config1.environmentAsTuples == [] + config2.environmentAsTuples == [new Tuple2('FOO','one'), new Tuple2('BAR','two')] + config3.environmentAsTuples == [new Tuple2('FOO',''), new Tuple2('BAR','')] + + } +} diff --git a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy index 358a1804c..7d6652fcd 100644 --- a/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/BuildControllerTest.groovy @@ -95,7 +95,7 @@ class BuildControllerTest extends Specification { final event = new BuildEvent(build, result) final entry = WaveBuildRecord.fromEvent(event) and: - persistenceService.saveBuild(entry) + persistenceService.saveBuildAsync(entry) when: def req = HttpRequest.GET("/v1alpha1/builds/${build.buildId}") def res = client.toBlocking().exchange(req, WaveBuildRecord) @@ -135,7 +135,7 @@ class BuildControllerTest extends Specification { requestIp: '127.0.0.1', startTime: Instant.now().minus(1, ChronoUnit.DAYS) ) and: - persistenceService.saveBuild(build1) + persistenceService.saveBuildAsync(build1) sleep(500) when: diff --git a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy index 1a4658f62..d582316b8 100644 --- a/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ContainerControllerTest.groovy @@ -72,7 +72,6 @@ import jakarta.inject.Inject * @author Paolo Di Tommaso */ @MicronautTest -@Property(name='wave.build.workspace', value='/some/wsp') @Property(name='wave.build.repo', value='wave/build') @Property(name='wave.build.cache', value='wave/build/cache') class ContainerControllerTest extends Specification { diff --git a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy index ef37f10ba..9057ac53e 100644 --- a/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/RegistryControllerLocalTest.groovy @@ -142,16 +142,19 @@ class RegistryControllerLocalTest extends Specification implements DockerRegistr h.add('Accept', it) } }) - HttpResponse response = client.toBlocking().exchange(request, Map) + HttpResponse response = client.toBlocking().exchange(request, String) then: response.status() == HttpStatus.OK when: - def list = response.body().manifests.collect{ - String type = it.mediaType.indexOf("manifest") ? "manifests" : "blobs" - "/v2/$IMAGE/$type/$it.digest" + def parsedBody = new JsonSlurper().parseText(response.body.get()) as Map + and: + def list = parsedBody.manifests.collect { + String type = it.mediaType.contains("manifest") ? "manifests" : "blobs" + return "/v2/$IMAGE/$type/$it.digest" as String } + and: boolean fails = list.find{ url -> HttpRequest requestGet = HttpRequest.GET(url).headers({ h -> accept.each { diff --git a/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy index c0b52f051..5677bf77a 100644 --- a/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ScanControllerTest.groovy @@ -79,7 +79,7 @@ class ScanControllerTest extends Specification { 'SUCCEEDED', results)) and: - persistenceService.saveScanRecord(scan) + persistenceService.saveScanRecordAsync(scan) when: def req = HttpRequest.GET("/v1alpha1/scans/${scan.id}") diff --git a/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy index 5f3cad31f..586d48616 100644 --- a/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ValidateCredsControllerTest.groovy @@ -67,9 +67,9 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'should validate username required'() { when: - HttpRequest request = HttpRequest.POST("/validate-creds", [ + HttpRequest request = HttpRequest.POST("/validate-creds", [request: [ password: 'test', - ]) + ]]) client.toBlocking().exchange(request, Boolean) then: def e = thrown(HttpClientResponseException) @@ -77,9 +77,9 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'should validate pwd required'() { when: - HttpRequest request = HttpRequest.POST("/validate-creds", [ + HttpRequest request = HttpRequest.POST("/validate-creds", [request:[ userName: 'test', - ]) + ]]) client.toBlocking().exchange(request, Boolean) then: def e = thrown(HttpClientResponseException) @@ -87,10 +87,10 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'should validate the test user'() { given: - def req = [ + def req = [request: [ userName:'test', password:'test', - registry: getTestRegistryUrl('test') ] + registry: getTestRegistryUrl('test') ]] and: HttpRequest request = HttpRequest.POST("/validate-creds", req) when: @@ -103,11 +103,11 @@ class ValidateCredsControllerTest extends Specification implements SecureDockerR void 'test validateController valid login'() { given: - def req = [ + def req = [request: [ userName: USER, password: PWD, registry: getTestRegistryUrl(REGISTRY_URL) - ] + ]] HttpRequest request = HttpRequest.POST("/validate-creds", req) when: HttpResponse response = client.toBlocking().exchange(request, Boolean) diff --git a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy index b7dafe358..ae904b5b9 100644 --- a/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy +++ b/src/test/groovy/io/seqera/wave/controller/ViewControllerTest.groovy @@ -55,6 +55,9 @@ import io.seqera.wave.tower.User import jakarta.inject.Inject import static io.seqera.wave.util.DataTimeUtils.formatDuration import static io.seqera.wave.util.DataTimeUtils.formatTimestamp + +import static io.seqera.wave.controller.ViewController.Colour + /** * * @author Paolo Di Tommaso @@ -147,7 +150,7 @@ class ViewControllerTest extends Specification { exitStatus: 0 ) when: - persistenceService.saveBuild(record1) + persistenceService.saveBuildAsync(record1) and: def request = HttpRequest.GET("/view/builds/${record1.buildId}") def response = client.toBlocking().exchange(request, String) @@ -177,7 +180,7 @@ class ViewControllerTest extends Specification { exitStatus: 0 ) when: - persistenceService.saveBuild(record1) + persistenceService.saveBuildAsync(record1) and: def request = HttpRequest.GET("/view/builds/${record1.buildId}") def response = client.toBlocking().exchange(request, String) @@ -219,7 +222,7 @@ class ViewControllerTest extends Specification { def container = new WaveContainerRecord(req, data, wave, addr, exp) when: - persistenceService.saveContainerRequest(container) + persistenceService.saveContainerRequestAsync(container) and: def request = HttpRequest.GET("/view/containers/${token}") def response = client.toBlocking().exchange(request, String) @@ -395,7 +398,7 @@ class ViewControllerTest extends Specification { ) when: - persistenceService.saveScanRecord(scan) + persistenceService.saveScanRecordAsync(scan) and: def request = HttpRequest.GET("/view/scans/${scan.id}") def response = client.toBlocking().exchange(request, String) @@ -429,7 +432,7 @@ class ViewControllerTest extends Specification { ) when: - persistenceService.saveMirrorResult(record1) + persistenceService.saveMirrorResultAsync(record1) and: def request = HttpRequest.GET("/view/mirrors/${record1.mirrorId}") def response = client.toBlocking().exchange(request, String) @@ -488,9 +491,9 @@ class ViewControllerTest extends Specification { exitStatus: 0 ) and: - persistenceService.saveBuild(record1) - persistenceService.saveBuild(record2) - persistenceService.saveBuild(record3) + persistenceService.saveBuildAsync(record1) + persistenceService.saveBuildAsync(record2) + persistenceService.saveBuildAsync(record3) when: def request = HttpRequest.GET("/view/builds/0727765dc72cee24") @@ -541,7 +544,7 @@ class ViewControllerTest extends Specification { exitStatus: 0 ) when: - persistenceService.saveBuild(record1) + persistenceService.saveBuildAsync(record1) and: def request = HttpRequest.GET("/view/builds/112233-1") def response = client.toBlocking().exchange(request, String) @@ -715,8 +718,8 @@ class ViewControllerTest extends Specification { def scan2 = new WaveScanRecord('sc-1234567890abcde_2', '101', null, null, CONTAINER_IMAGE, PLATFORM, Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null) when: - persistenceService.saveScanRecord(scan1) - persistenceService.saveScanRecord(scan2) + persistenceService.saveScanRecordAsync(scan1) + persistenceService.saveScanRecordAsync(scan2) and: def request = HttpRequest.GET("/view/scans/1234567890abcde") def response = client.toBlocking().exchange(request, String) @@ -743,4 +746,21 @@ class ViewControllerTest extends Specification { null | false '1234567890abcdef' | true } + + @Unroll + def 'should return correct scan color based on vulnerabilities'() { + expect: + ViewController.getScanColor(VULNERABILITIES) == EXPEXTED_COLOR + + where: + VULNERABILITIES | EXPEXTED_COLOR + [new ScanVulnerability(severity: 'LOW')] | new Colour('#dff0d8','#3c763d') + [new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#fff8c5','#000000') + [new ScanVulnerability(severity: 'HIGH')] | new Colour('#ffe4e2','#e00404') + [new ScanVulnerability(severity: 'CRITICAL')] | new Colour('#ffe4e2','#e00404') + [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'MEDIUM')] | new Colour('#fff8c5','#000000') + [new ScanVulnerability(severity: 'LOW'), new ScanVulnerability(severity: 'HIGH')] | new Colour('#ffe4e2','#e00404') + [new ScanVulnerability(severity: 'MEDIUM'), new ScanVulnerability(severity: 'CRITICAL')] | new Colour('#ffe4e2','#e00404') + [] | new Colour('#dff0d8','#3c763d') + } } diff --git a/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy b/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy index 44d2db027..68674e2fb 100644 --- a/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/core/RegistryProxyServiceTest.groovy @@ -57,7 +57,7 @@ class RegistryProxyServiceTest extends Specification { @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) def 'should retrieve image digest on ECR' () { given: - def IMAGE = '195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/kaniko:0.1.0' + def IMAGE = '195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/app:1.0.0' def request = Mock(BuildRequest) when: @@ -66,6 +66,6 @@ class RegistryProxyServiceTest extends Specification { request.getTargetImage() >> IMAGE request.getIdentity() >> new PlatformId() then: - resp1 == 'sha256:05f9dc67e6ec879773de726b800d4d5044f8bd8e67da728484fbdea56af1fdff' + resp1 == 'sha256:d5c169e210d6b789b2dc7eced66471cf4ce2c7260ac7299fbef464ff902086be' } } diff --git a/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy b/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy index a3e3328e3..f952cf4e8 100644 --- a/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy +++ b/src/test/groovy/io/seqera/wave/proxy/ProxyClientTest.groovy @@ -145,7 +145,7 @@ class ProxyClientTest extends Specification { @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) def 'should call target manifest on amazon' () { given: - def IMAGE = 'wave/kaniko' + def IMAGE = 'wave/app' def REG = '195996028523.dkr.ecr.eu-west-1.amazonaws.com' def registry = lookupService.lookup(REG) def creds = credentialsProvider.getDefaultCredentials(REG) @@ -158,7 +158,7 @@ class ProxyClientTest extends Specification { .withCredentials(creds) when: - def resp = proxy.getString("/v2/$IMAGE/manifests/0.1.0") + def resp = proxy.getString("/v2/$IMAGE/manifests/1.0.0") then: resp.statusCode() == 200 } @@ -166,7 +166,7 @@ class ProxyClientTest extends Specification { @Requires({System.getenv('AWS_ACCESS_KEY_ID') && System.getenv('AWS_SECRET_ACCESS_KEY')}) def 'should call head manifest on amazon' () { given: - def IMAGE = 'wave/kaniko' + def IMAGE = 'wave/app' def REG = '195996028523.dkr.ecr.eu-west-1.amazonaws.com' def registry = lookupService.lookup(REG) def creds = credentialsProvider.getDefaultCredentials(REG) @@ -179,7 +179,7 @@ class ProxyClientTest extends Specification { .withCredentials(creds) when: - def resp = proxy.head("/v2/$IMAGE/manifests/0.1.0") + def resp = proxy.head("/v2/$IMAGE/manifests/1.0.0") then: resp.statusCode() == 200 } diff --git a/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy index a2babb87d..66b2012b2 100644 --- a/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/CredentialsServiceTest.groovy @@ -51,11 +51,9 @@ class CredentialsServiceTest extends Specification { @Inject CredentialsService credentialsService - @MockBean(TowerClient) TowerClient towerClient = Mock(TowerClient) - @MockBean(PairingService) PairingService securityService = Mock(PairingService) @@ -121,7 +119,6 @@ class CredentialsServiceTest extends Specification { noExceptionThrown() } - def 'should fail if keys where not registered for the tower endpoint'() { given: def identity = new PlatformId(new User(id:10), 10,"token",'endpoint') @@ -169,7 +166,7 @@ class CredentialsServiceTest extends Specification { registry: 'docker.io' ) and: - def identity = new PlatformId(new User(id:10), 10,"token",'tower.io', '101') + def identity = new PlatformId(new User(id:10), 100, 'token', 'tower.io', '101') def auth = JwtAuth.of(identity) when: @@ -185,7 +182,7 @@ class CredentialsServiceTest extends Specification { ) and: 'non matching credentials are listed' - 1 * towerClient.listCredentials('tower.io',auth,10) >> CompletableFuture.completedFuture(new ListCredentialsResponse( + 1 * towerClient.listCredentials('tower.io',auth,100) >> CompletableFuture.completedFuture(new ListCredentialsResponse( credentials: [nonContainerRegistryCredentials,otherRegistryCredentials] )) @@ -223,7 +220,7 @@ class CredentialsServiceTest extends Specification { def 'should get registry creds from compute creds when not found in tower credentials'() { given: 'a tower user in a workspace on a specific instance with a valid token' def userId = 10 - def workspaceId = 10 + def workspaceId = 100 def token = "valid-token" def towerEndpoint = "http://tower.io:9090" def workflowId = "id123" diff --git a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy index 1dce14f26..83fab8470 100644 --- a/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/blob/impl/KubeTransferStrategyTest.groovy @@ -25,7 +25,6 @@ import java.time.OffsetDateTime import io.kubernetes.client.openapi.models.V1Job import io.kubernetes.client.openapi.models.V1Pod -import io.kubernetes.client.openapi.models.V1PodList import io.kubernetes.client.openapi.models.V1PodStatus import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.service.blob.BlobEntry @@ -48,11 +47,8 @@ class KubeTransferStrategyTest extends Specification { def podName = "$jobName-abc".toString() def pod = new V1Pod(metadata: [name: podName, creationTimestamp: OffsetDateTime.now()]) pod.status = new V1PodStatus(phase: "Succeeded") - def podList = new V1PodList(items: [pod]) k8sService.launchTransferJob(_, _, _, _) >> new V1Job(metadata: [name: jobName]) - k8sService.waitJob(_, _) >> podList k8sService.getPod(_) >> pod - k8sService.waitPodCompletion(_, _) >> 0 k8sService.logsPod(_) >> "transfer successful" when: diff --git a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy index a7a0cc6b0..e422a87f6 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/ContainerBuildServiceTest.groovy @@ -49,6 +49,7 @@ import io.seqera.wave.service.job.JobService import io.seqera.wave.service.job.JobSpec import io.seqera.wave.service.job.JobState import io.seqera.wave.service.persistence.PersistenceService +import io.seqera.wave.service.scan.ContainerScanService import io.seqera.wave.service.scan.ScanRequest import io.seqera.wave.service.scan.ScanStrategy import io.seqera.wave.test.TestHelper @@ -323,10 +324,10 @@ class ContainerBuildServiceTest extends Specification { and: def result = new BuildResult(request.buildId, 0, "content", Instant.now(), Duration.ofSeconds(1), 'abc123') - def event = new BuildEvent(request, result) + def entry = new BuildEntry(request, result) when: - service.onBuildEvent(event) + service.handleBuildCompletion(entry) then: def record = service.getBuildRecord(request.buildId) @@ -347,10 +348,12 @@ class ContainerBuildServiceTest extends Specification { def 'should handle job completion event and update build store'() { given: - def mockBuildStore = Mock(BuildStateStore) - def mockProxyService = Mock(RegistryProxyService) - def mockEventPublisher = Mock(ApplicationEventPublisher) - def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def buildStore = Mock(BuildStateStore) + def proxyService = Mock(RegistryProxyService) + def persistenceService = Mock(PersistenceService) + def scanService = Mock(ContainerScanService) + def eventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, proxyService: proxyService, eventPublisher: eventPublisher, persistenceService: persistenceService, scanService:scanService, buildConfig: buildConfig) def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def state = JobState.succeeded('logs') def res = BuildResult.create('1') @@ -358,7 +361,8 @@ class ContainerBuildServiceTest extends Specification { targetImage: 'docker.io/foo:0', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + identity: PlatformId.NULL ) def build = new BuildEntry(req, res) @@ -366,19 +370,24 @@ class ContainerBuildServiceTest extends Specification { service.onJobCompletion(job, build, state) then: - 1 * mockBuildStore.storeBuild('1', _) + 1 * scanService.scanOnBuild(_) >> null and: - 1 * mockProxyService.getImageDigest(_, _) >> 'digest' + 1 * buildStore.storeBuild(req.targetImage, _) >> null and: - 1 * mockEventPublisher.publishEvent(_) + 1 * proxyService.getImageDigest(_, _) >> 'digest' + and: + 1 * persistenceService.saveBuildAsync(_) >> null + and: + 1 * eventPublisher.publishEvent(_) } def 'should handle job error event and update build store'() { given: - def mockBuildStore = Mock(BuildStateStore) - def mockProxyService = Mock(RegistryProxyService) - def mockEventPublisher = Mock(ApplicationEventPublisher) - def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def buildStore = Mock(BuildStateStore) + def proxyService = Mock(RegistryProxyService) + def persistenceService = Mock(PersistenceService) + def eventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, proxyService: proxyService, eventPublisher: eventPublisher, persistenceService:persistenceService, buildConfig: buildConfig) def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def error = new Exception('error') def res = BuildResult.create('1') @@ -386,7 +395,8 @@ class ContainerBuildServiceTest extends Specification { targetImage: 'docker.io/foo:0', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + identity: PlatformId.NULL ) def build = new BuildEntry(req, res) @@ -394,24 +404,28 @@ class ContainerBuildServiceTest extends Specification { service.onJobException(job, build, error) then: - 1 * mockBuildStore.storeBuild('1', _) + 1 * buildStore.storeBuild(req.targetImage, _) >> null and: - 1 * mockEventPublisher.publishEvent(_) + 1 * persistenceService.saveBuildAsync(_) >> null + and: + 1 * eventPublisher.publishEvent(_) } def 'should handle job timeout event and update build store'() { given: - def mockBuildStore = Mock(BuildStateStore) - def mockProxyService = Mock(RegistryProxyService) - def mockEventPublisher = Mock(ApplicationEventPublisher) - def service = new ContainerBuildServiceImpl(buildStore: mockBuildStore, proxyService: mockProxyService, eventPublisher: mockEventPublisher, buildConfig: buildConfig) + def buildStore = Mock(BuildStateStore) + def proxyService = Mock(RegistryProxyService) + def persistenceService = Mock(PersistenceService) + def eventPublisher = Mock(ApplicationEventPublisher) + def service = new ContainerBuildServiceImpl(buildStore: buildStore, proxyService: proxyService, eventPublisher: eventPublisher, persistenceService:persistenceService, buildConfig: buildConfig) def job = JobSpec.build('1', 'operationName', Instant.now(), Duration.ofMinutes(1), Path.of('/work/dir')) def res = BuildResult.create('1') def req = new BuildRequest( targetImage: 'docker.io/foo:0', buildId: '1', startTime: Instant.now(), - maxDuration: Duration.ofMinutes(1) + maxDuration: Duration.ofMinutes(1), + identity: PlatformId.NULL ) def build = new BuildEntry(req, res) @@ -419,9 +433,11 @@ class ContainerBuildServiceTest extends Specification { service.onJobTimeout(job, build) then: - 1 * mockBuildStore.storeBuild('1', _) + 1 * buildStore.storeBuild(req.targetImage, _) >> null + and: + 1 * persistenceService.saveBuildAsync(_) >> null and: - 1 * mockEventPublisher.publishEvent(_) + 1 * eventPublisher.publishEvent(_) } } diff --git a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy index 74ffaae38..2c9e221e2 100644 --- a/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/builder/KubeBuildStrategyTest.groovy @@ -38,11 +38,9 @@ import jakarta.inject.Inject * @author Paolo Di Tommaso */ @MicronautTest -@Property(name="wave.build.workspace",value="/build/work") @Property(name="wave.build.k8s.namespace",value="foo") @Property(name="wave.build.k8s.configPath",value="/home/kube.config") @Property(name="wave.build.k8s.storage.claimName",value="bar") -@Property(name="wave.build.k8s.storage.mountPath",value="/build") @Property(name='wave.build.k8s.node-selector[linux/amd64]',value="service=wave-build") @Property(name='wave.build.k8s.node-selector[linux/arm64]',value="service=wave-build-arm64") class KubeBuildStrategyTest extends Specification { diff --git a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy index f58513a13..04ea45d71 100644 --- a/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/job/JobManagerTest.groovy @@ -26,7 +26,6 @@ import java.time.Instant import com.github.benmanes.caffeine.cache.Cache import com.github.benmanes.caffeine.cache.Caffeine - /** * * @author Munish Chouhan @@ -38,7 +37,7 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def cache = Caffeine.newBuilder().build() + def cache = Caffeine.newBuilder().buildAsync() def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now(), Duration.ofMinutes(10)) @@ -57,7 +56,8 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config) + def cache = Caffeine.newBuilder().buildAsync() + def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now(), Duration.ofMinutes(10)) @@ -75,7 +75,7 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def cache = Caffeine.newBuilder().build() + def cache = Caffeine.newBuilder().buildAsync() def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config:config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now() - Duration.ofMinutes(5), Duration.ofMinutes(2)) @@ -94,7 +94,7 @@ class JobManagerTest extends Specification { def jobService = Mock(JobService) def jobDispatcher = Mock(JobDispatcher) def config = new JobConfig(graceInterval: Duration.ofMillis(1)) - def cache = Caffeine.newBuilder().build() + def cache = Caffeine.newBuilder().buildAsync() def manager = new JobManager(jobService: jobService, dispatcher: jobDispatcher, config: config, debounceCache: cache) and: def jobSpec = JobSpec.transfer('foo', 'scheduler-1', Instant.now().minus(Duration.ofMillis(500)), Duration.ofMinutes(10)) diff --git a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy index 03d136e19..f1b276c47 100644 --- a/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/k8s/K8sServiceImplTest.groovy @@ -36,28 +36,16 @@ import io.kubernetes.client.openapi.models.V1JobStatus import io.kubernetes.client.openapi.models.V1ObjectMeta import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList -import io.kubernetes.client.openapi.models.V1PodStatus import io.micronaut.context.ApplicationContext -import io.micronaut.context.annotation.Replaces -import io.micronaut.test.extensions.spock.annotation.MicronautTest import io.seqera.wave.configuration.BlobCacheConfig -import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.configuration.MirrorConfig +import io.seqera.wave.configuration.ScanConfig /** * * @author Paolo Di Tommaso */ -@MicronautTest class K8sServiceImplTest extends Specification { - @Replaces(ScanConfig.class) - static class MockScanConfig extends ScanConfig { - @Override - Path getCacheDirectory() { - return Path.of('/build/scan/cache') - } - } - def 'should validate context OK ' () { when: def PROPS = [ @@ -522,58 +510,6 @@ class K8sServiceImplTest extends Specification { ctx.close() } - def "deletePodWhenReachStatus should delete pod when status is reached within timeout"() { - given: - def podName = "test-pod" - def statusName = "Succeeded" - def timeout = 5000 - def api = Mock(CoreV1Api) - api.readNamespacedPod(_,_,_) >> new V1Pod(status: new V1PodStatus(phase: statusName)) - def k8sClient = new K8sClient() { - @Override - ApiClient apiClient() { - return null - } - CoreV1Api coreV1Api() { - return api - } - } - - def k8sService = new K8sServiceImpl(k8sClient: k8sClient) - - when: - k8sService.deletePodWhenReachStatus(podName, statusName, timeout) - - then: - 1 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) - } - - def "deletePodWhenReachStatus should not delete pod if status is not reached within timeout"() { - given: - def podName = "test-pod" - def statusName = "Succeeded" - def timeout = 5000 - def api = Mock(CoreV1Api) - api.readNamespacedPod(_,_,_) >> new V1Pod(status: new V1PodStatus(phase: "Running")) - def k8sClient = new K8sClient() { - @Override - ApiClient apiClient() { - return null - } - CoreV1Api coreV1Api() { - return api - } - } - - def k8sService = new K8sServiceImpl(k8sClient: k8sClient) - - when: - k8sService.deletePodWhenReachStatus(podName, statusName, timeout) - - then: - 0 * api.deleteNamespacedPod('test-pod', null, null, null, null, null, null, null) - } - def "getLatestPodForJob should return the latest pod when multiple pods are present"() { given: def jobName = "test-job" @@ -748,7 +684,7 @@ class K8sServiceImplTest extends Specification { getCacheDirectory() >> Path.of('/build/cache/dir') getRequestsCpu() >> '2' getRequestsMemory() >> '4Gi' - getGithubToken() >> '123abc' + getEnvironmentAsTuples() >> [new Tuple2('FOO', 'abc'), new Tuple2('BAR', 'xyz')] } when: @@ -761,7 +697,7 @@ class K8sServiceImplTest extends Specification { job.spec.template.spec.containers[0].args == args job.spec.template.spec.containers[0].resources.requests.get('cpu') == new Quantity('2') job.spec.template.spec.containers[0].resources.requests.get('memory') == new Quantity('4Gi') - job.spec.template.spec.containers[0].env == [ new V1EnvVar().name('GITHUB_TOKEN').value('123abc') ] + job.spec.template.spec.containers[0].env == [ new V1EnvVar().name('FOO').value('abc'), new V1EnvVar().name('BAR').value('xyz') ] job.spec.template.spec.volumes.size() == 1 job.spec.template.spec.volumes[0].persistentVolumeClaim.claimName == 'bar' job.spec.template.spec.restartPolicy == 'Never' diff --git a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy index d892af4f6..c9eedc079 100644 --- a/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/mirror/ContainerMirrorServiceTest.groovy @@ -105,7 +105,7 @@ class ContainerMirrorServiceTest extends Specification { and: def state = MirrorResult.of(request) and: - persistenceService.saveMirrorResult(state) + persistenceService.saveMirrorResultAsync(state) when: def copy = mirrorService.getMirrorResult(request.mirrorId) then: diff --git a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy index ea27f24b7..a8eb7c47b 100644 --- a/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/persistence/impl/SurrealPersistenceServiceTest.groovy @@ -128,7 +128,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe when: storage.initializeDb() and: - storage.saveBuild(build) + storage.saveBuildAsync(build) then: sleep 100 def stored = storage.loadBuild(request.buildId) @@ -160,7 +160,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def record = WaveBuildRecord.fromEvent(event) and: - persistence.saveBuild(record) + persistence.saveBuildAsync(record) when: sleep 100 @@ -248,7 +248,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) when: - persistence.saveBuild(build1) + persistence.saveBuildAsync(build1) sleep 100 then: persistence.loadBuild(request.buildId) == build1 @@ -281,7 +281,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe and: def request = new WaveContainerRecord(req, data, wave, addr, exp) and: - persistence.saveContainerRequest(request) + persistence.saveContainerRequestAsync(request) and: sleep 200 // <-- the above request is async, give time to save it @@ -293,7 +293,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe // should update the record when: - persistence.updateContainerRequest(TOKEN, new ContainerDigestPair('111', '222')) + persistence.updateContainerRequestAsync(TOKEN, new ContainerDigestPair('111', '222')) and: sleep 200 then: @@ -323,7 +323,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def CVE4 = new ScanVulnerability('cve-4', 'x4', 'title4', 'package4', 'version4', 'fixed4', 'url4') def scan = new WaveScanRecord(SCAN_ID, BUILD_ID, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(10), 'SUCCEEDED', [CVE1, CVE2, CVE3], null, null) when: - persistence.saveScanRecord(scan) + persistence.saveScanRecordAsync(scan) + sleep 200 then: def result = persistence.loadScanRecord(SCAN_ID) and: @@ -342,7 +343,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def scanRecord2 = new WaveScanRecord(SCAN_ID2, BUILD_ID2, null, null, CONTAINER_IMAGE, PLATFORM, NOW, Duration.ofSeconds(20), 'FAILED', [CVE1, CVE4], 1, "Error 'quote'") and: // should save the same CVE into another build - persistence.saveScanRecord(scanRecord2) + persistence.saveScanRecordAsync(scanRecord2) + sleep 200 then: def result2 = persistence.loadScanRecord(SCAN_ID2) and: @@ -371,7 +373,8 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe !persistence.existsScanRecord(SCAN_ID) when: - persistence.saveScanRecord(scan) + persistence.saveScanRecordAsync(scan) + sleep 200 then: persistence.existsScanRecord(SCAN_ID) } @@ -398,7 +401,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe storage.initializeDb() and: def result = MirrorEntry.of(request).getResult() - storage.saveMirrorResult(result) + storage.saveMirrorResultAsync(result) sleep 100 when: @@ -458,9 +461,9 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def result1 = MirrorResult.of(request1).complete(1, 'err') def result2 = MirrorResult.of(request2).complete(0, 'ok') def result3 = MirrorResult.of(request3).complete(0, 'ok') - storage.saveMirrorResult(result1) - storage.saveMirrorResult(result2) - storage.saveMirrorResult(result3) + storage.saveMirrorResultAsync(result1) + storage.saveMirrorResultAsync(result2) + storage.saveMirrorResultAsync(result3) sleep 100 when: @@ -506,7 +509,7 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def build1 = WaveBuildRecord.fromEvent(new BuildEvent(request, result)) when: - persistence.saveBuild(build1) + persistence.saveBuildAsync(build1) sleep 100 then: persistence.loadBuild(request.buildId) == build1 @@ -562,11 +565,12 @@ class SurrealPersistenceServiceTest extends Specification implements SurrealDBTe def scan4 = new WaveScanRecord('sc-01234567890abcdef_4', '103', null, null, CONTAINER_IMAGE, PLATFORM,Instant.now(), Duration.ofSeconds(10), 'SUCCEEDED', [CVE1], null, null) when: - persistence.saveScanRecord(scan1) - persistence.saveScanRecord(scan2) - persistence.saveScanRecord(scan3) - persistence.saveScanRecord(scan4) - + persistence.saveScanRecordAsync(scan1) + persistence.saveScanRecordAsync(scan2) + persistence.saveScanRecordAsync(scan3) + persistence.saveScanRecordAsync(scan4) + and: + sleep 200 then: persistence.allScans("1234567890abcdef") == [scan3, scan2, scan1] and: diff --git a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy index 59d825dea..05fe4efac 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/ContainerScanServiceImplTest.groovy @@ -27,6 +27,7 @@ import java.time.Duration import java.time.Instant import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.api.ScanMode import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.builder.BuildFormat @@ -150,7 +151,7 @@ class ContainerScanServiceImplTest extends Specification { and: def KEY = 'scan-20' def jobService = Mock(JobService) - def service = new ContainerScanServiceImpl(scanStore: scanStore, persistenceService: persistenceService, jobService: jobService) + def service = new ContainerScanServiceImpl(scanStore: scanStore, persistenceService: persistenceService, jobService: jobService, config: new ScanConfig(vulnerabilityLimit: 100)) def job = JobSpec.scan(KEY, 'ubuntu:latest', Instant.now(), Duration.ofMinutes(1), workDir) def scan = ScanEntry.of(scanId: KEY, buildId: 'build-20', containerImage: 'ubuntu:latest', startTime: Instant.now()) @@ -367,6 +368,7 @@ class ContainerScanServiceImplTest extends Specification { def scanService = Spy(new ContainerScanServiceImpl(inspectService: inspectService, config: config)) def request = Mock(ContainerRequest) request.scanId >> SCAN_ID + request.scanMode >> MODE request.isContainer() >> CONTAINER request.dryRun >> DRY_RUN and: @@ -379,12 +381,14 @@ class ContainerScanServiceImplTest extends Specification { RUN_TIMES * scanService.scan(scan) >> null where: - SCAN_ID | CONTAINER | DRY_RUN | RUN_TIMES - null | false | false | 0 - 'sc-123'| false | false | 0 - 'sc-123'| true | false | 1 - 'sc-123'| true | true | 0 - null | true | false | 0 + SCAN_ID | MODE | CONTAINER | DRY_RUN | RUN_TIMES + null | ScanMode.async | false | false | 0 + 'sc-123'| ScanMode.async | false | false | 0 + 'sc-123'| ScanMode.async | true | false | 1 + 'sc-123'| ScanMode.required | true | false | 1 + 'sc-123'| ScanMode.none | true | false | 0 + 'sc-123'| ScanMode.async | true | true | 0 + null | ScanMode.async | true | false | 0 } @@ -398,10 +402,12 @@ class ContainerScanServiceImplTest extends Specification { scanService.existsScan(SCAN_ID) >> EXISTS_SCAN and: def request = Mock(ContainerRequest) + request.scanMode >> MODE request.scanId >> SCAN_ID request.buildId >> BUILD_ID request.buildNew >> BUILD_NEW request.dryRun >> DRY_RUN + request.succeeded >> SUCCEEDED and: def scan = Mock(ScanRequest) @@ -412,15 +418,18 @@ class ContainerScanServiceImplTest extends Specification { RUN_TIMES * scanService.scan(scan) >> null where: - SCAN_ID | BUILD_ID | BUILD_NEW | DRY_RUN | EXISTS_SCAN | RUN_TIMES - null | null | null | null | false | 0 - 'sc-123'| null | null | null | false | 0 + SCAN_ID | BUILD_ID | BUILD_NEW | SUCCEEDED | MODE | DRY_RUN | EXISTS_SCAN | RUN_TIMES + null | null | null | null | ScanMode.async | null | false | 0 + 'sc-123'| null | null | null | ScanMode.async | null | false | 0 and: - 'sc-123'| 'bd-123' | null | null | false | 0 - 'sc-123'| 'bd-123' | true | null | false | 0 - 'sc-123'| 'bd-123' | false | null | false | 1 - 'sc-123'| 'bd-123' | false | null | true | 0 - 'sc-123'| 'bd-123' | false | true | false | 0 + 'sc-123'| 'bd-123' | null | null | ScanMode.async | null | false | 0 + 'sc-123'| 'bd-123' | true | null | ScanMode.async | null | false | 0 + 'sc-123'| 'bd-123' | false | true | ScanMode.async | null | false | 1 + 'sc-123'| 'bd-123' | false | true | ScanMode.required | null | false | 1 + 'sc-123'| 'bd-123' | false | true | ScanMode.none | null | false | 0 + 'sc-123'| 'bd-123' | false | false | ScanMode.async | null | false | 0 + 'sc-123'| 'bd-123' | false | null | ScanMode.async | null | true | 0 + 'sc-123'| 'bd-123' | false | null | ScanMode.async | true | false | 0 } def 'should store scan entry' () { @@ -443,21 +452,21 @@ class ContainerScanServiceImplTest extends Specification { scanService.storeScanEntry(scanSucceeded) then: 1 * scanStore.storeScan(scanSucceeded) >> null - 1 * persistenceService.saveScanRecord(new WaveScanRecord(scanSucceeded)) >> null + 1 * persistenceService.saveScanRecordAsync(new WaveScanRecord(scanSucceeded)) >> null 0 * cleanupService.cleanupScanId(container) >> null when: scanService.storeScanEntry(scanNotDone) then: 1 * scanStore.storeScan(scanNotDone) >> null - 1 * persistenceService.saveScanRecord(new WaveScanRecord(scanNotDone)) >> null + 1 * persistenceService.saveScanRecordAsync(new WaveScanRecord(scanNotDone)) >> null 0 * cleanupService.cleanupScanId(container) >> null when: scanService.storeScanEntry(scanFailed) then: 1 * scanStore.storeScan(scanFailed) >> null - 1 * persistenceService.saveScanRecord(new WaveScanRecord(scanFailed)) >> null + 1 * persistenceService.saveScanRecordAsync(new WaveScanRecord(scanFailed)) >> null 1 * cleanupService.cleanupScanId(container) >> null } } diff --git a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy index f490e0aa1..1a759cc58 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/DockerScanStrategyTest.groovy @@ -20,29 +20,35 @@ package io.seqera.wave.service.scan import spock.lang.Specification -import java.nio.file.Files import java.nio.file.Path -import io.micronaut.context.ApplicationContext +import io.micronaut.test.annotation.MockBean +import io.micronaut.test.extensions.spock.annotation.MicronautTest +import io.seqera.wave.configuration.ScanConfig +import jakarta.inject.Inject /** * * @author Munish Chouhan */ +@MicronautTest class DockerScanStrategyTest extends Specification { + @Inject + DockerScanStrategy dockerContainerStrategy + + @MockBean(ScanConfig) + ScanConfig mockConfig() { + Mock(ScanConfig) { + getCacheDirectory() >> Path.of('/some/scan/cache') + } + } + def 'should get docker command' () { - given: - def workspace = Files.createTempDirectory('test') - def props = ['wave.build.workspace': workspace.toString()] - and: - def ctx = ApplicationContext.run(props) - and: - def dockerContainerStrategy = ctx.getBean(DockerScanStrategy) when: def scanDir = Path.of('/some/scan/dir') def config = Path.of("/user/test/build-workspace/config.json") - def command = dockerContainerStrategy.dockerWrapper('foo-123', scanDir, config, 'xyz') + def command = dockerContainerStrategy.dockerWrapper('foo-123', scanDir, config, ['FOO=1', 'BAR=2']) then: command == [ @@ -56,15 +62,14 @@ class DockerScanStrategyTest extends Specification { '-v', '/some/scan/dir:/some/scan/dir:rw', '-v', - "/build/scan/cache:/root/.cache/:rw", + '/some/scan/cache:/root/.cache/:rw', '-v', '/user/test/build-workspace/config.json:/root/.docker/config.json:ro', '-e', - 'GITHUB_TOKEN=xyz' + 'FOO=1', + '-e', + 'BAR=2' ] - cleanup: - ctx.close() - workspace?.deleteDir() } } diff --git a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy index 0563f8842..af9b4b549 100644 --- a/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/scan/TrivyResultProcessorTest.groovy @@ -18,11 +18,11 @@ package io.seqera.wave.service.scan - import spock.lang.Specification -import io.seqera.wave.exception.ScanRuntimeException +import java.nio.file.Files +import io.seqera.wave.exception.ScanRuntimeException /** * * @author Munish Chouhan @@ -91,7 +91,7 @@ class TrivyResultProcessorTest extends Specification { """ when: - def result = TrivyResultProcessor.process(trivyDockerResulJson) + def result = TrivyResultProcessor.parseJson(trivyDockerResulJson) then: def vulnerability = result[0] @@ -105,9 +105,100 @@ class TrivyResultProcessorTest extends Specification { } + def "should return a sorted map of vulnerabilities"() { + given: + def folder = Files.createTempDirectory('test') + def scan = folder.resolve('scan.json') + and: + scan.text = """ + { "Results": [ + { + "Target": "sample-application", + "Class": "os-pkgs", + "Type": "linux", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2023-0001", + "PkgID": "example-lib@1.0.0", + "PkgName": "example-lib", + "InstalledVersion": "1.0.0", + "FixedVersion": "1.0.1", + "Severity": "LOW", + "Description": "A minor vulnerability with low impact.", + "PrimaryURL": "https://example.com/CVE-2023-0001" + }, + { + "VulnerabilityID": "CVE-2023-0002", + "PkgID": "example-lib@1.2.3", + "PkgName": "example-lib", + "InstalledVersion": "1.2.3", + "FixedVersion": "1.2.4", + "Severity": "MEDIUM", + "Description": "A vulnerability that allows unauthorized access.", + "PrimaryURL": "https://example.com/CVE-2023-0002" + }, + { + "VulnerabilityID": "CVE-2023-0003", + "PkgID": "example-lib@2.3.4", + "PkgName": "example-lib", + "InstalledVersion": "2.3.4", + "FixedVersion": "2.3.5", + "Severity": "HIGH", + "Description": "A vulnerability that could lead to remote code execution.", + "PrimaryURL": "https://example.com/CVE-2023-0003" + }, + { + "VulnerabilityID": "CVE-2023-0004", + "PkgID": "example-lib@3.0.0", + "PkgName": "example-lib", + "InstalledVersion": "3.0.0", + "FixedVersion": "3.0.1", + "Severity": "HIGH", + "Description": "A random test vulnerability with unspecified impact.", + "PrimaryURL": "https://example.com/CVE-2023-0004" + }, + { + "VulnerabilityID": "CVE-2023-0005", + "PkgID": "example-lib@3.1.0", + "PkgName": "example-lib", + "InstalledVersion": "3.1.0", + "FixedVersion": "3.1.1", + "Severity": "CRITICAL", + "Description": "Another random test vulnerability for testing purposes.", + "PrimaryURL": "https://example.com/CVE-2023-0005" + } + ] + } + ] + }""".stripIndent() + + when: + def topIssues = TrivyResultProcessor.parseFile(scan, 2) + + then: + topIssues.size() == 2 + topIssues[0].severity == "CRITICAL" + topIssues[0].id == "CVE-2023-0005" + topIssues[1].severity == "HIGH" + topIssues[1].id == "CVE-2023-0003" + + when: + def allIssues = TrivyResultProcessor.parseFile(scan) + then: + allIssues.size() == 5 + + cleanup: + folder?.deleteDir() + } + + def 'should not fail with empty list' () { + expect: + TrivyResultProcessor.filter([], 10) == [] + } + def "process should throw exception if json is not correct"() { when: - TrivyResultProcessor.process("invalid json") + TrivyResultProcessor.parseJson("invalid json") then: thrown ScanRuntimeException } diff --git a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy index d0316fb0e..cc0174184 100644 --- a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceProdTest.groovy @@ -73,7 +73,7 @@ class ValidationServiceProdTest extends Specification { ENDPOINT | EXPECTED 'foo' | "Missing endpoint protocol — offending value: foo" 'ftp://foo.com' | "Invalid endpoint protocol — offending value: ftp://foo.com" - 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 7: http://a b c" + 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 8: http://a b c" 'http://localhost' | 'Endpoint hostname not allowed — offending value: http://localhost' 'http://localhost:8000' | 'Endpoint hostname not allowed — offending value: http://localhost:8000' 'http://10.0.0.0/api' | 'Endpoint hostname not allowed — offending value: http://10.0.0.0/api' diff --git a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy index 12ba6c1e1..4ed710744 100644 --- a/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/validation/ValidationServiceTest.groovy @@ -54,7 +54,7 @@ class ValidationServiceTest extends Specification { ENDPOINT | EXPECTED 'foo' | "Missing endpoint protocol — offending value: foo" 'ftp://foo.com' | "Invalid endpoint protocol — offending value: ftp://foo.com" - 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 7: http://a b c" + 'http://a b c' | "Invalid endpoint 'http://a b c' — cause: Illegal character in authority at index 8: http://a b c" and: 'http://foo.com' | null 'http://localhost' | null