From 24c9dc4b5b199bc3506f3f1aa0da9be0a9238cc4 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 21 Aug 2019 09:57:17 +0200 Subject: [PATCH 01/78] Use OpenJDK Co-authored-by: Marco Geweke --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8bdd1c449..ca871d96d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: java jdk: - openjdk8 - - oraclejdk11 + - openjdk11 install: /bin/true # skip gradle assemble From 7427ac3ca7cebeaa5c21a669409933006e15d933 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 21 Aug 2019 13:18:46 +0200 Subject: [PATCH 02/78] Update readme, remove waffle.io links because waffle.io is gone :( Co-authored-by: Marco Geweke --- README.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index 9e5c96881..47e5b37f2 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,8 @@ Collection of independent libraries on top of Spring Boot to provide a faster se > "I never did anything by accident, nor did any of my inventions come by accident; they came by work." - Thomas Edison - ## Status -[![Next Selected Stories](https://badge.waffle.io/otto-de/edison-microservice.svg?label=Ready&title=Selected)](http://waffle.io/otto-de/edison-microservice) -[![Active Stories](https://badge.waffle.io/otto-de/edison-microservice.svg?label=In%20Progress&title=Doing)](http://waffle.io/otto-de/edison-microservice) - [![build](https://travis-ci.org/otto-de/edison-microservice.svg)](https://travis-ci.org/otto-de/edison-microservice) [![codecov](https://codecov.io/gh/otto-de/edison-microservice/branch/master/graph/badge.svg)](https://codecov.io/gh/otto-de/edison-microservice) [![Known Vulnerabilities](https://snyk.io/test/github/otto-de/edison-microservice/badge.svg)](https://snyk.io/test/github/otto-de/edison-microservice) @@ -18,7 +14,6 @@ Collection of independent libraries on top of Spring Boot to provide a faster se Have a look at the [release notes](CHANGELOG.md) for details about updates and changes. - ## About This project contains a number of independent libraries on top of Spring Boot to provide a faster setup of jvm microservices. @@ -45,7 +40,7 @@ This project maintains its roadmap with [issues](https://github.com/otto-de/edis **[1.x.0](https://github.com/otto-de/edison-microservice/milestone/2)**: Edison Microservices for Spring Boot 1.5 ✔ -**[2.0.0](https://github.com/otto-de/edison-microservice/milestone/3)**: Edison Microservices for Spring Boot 2.0 +**[2.0.0](https://github.com/otto-de/edison-microservice/milestone/3)**: Edison Microservices for Spring Boot 2.x ## Migration from Edison 1.x to Edison 2 From 8963bc3e473c462ca9c8446cc04d9bbf44fe75c4 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 21 Aug 2019 14:01:40 +0200 Subject: [PATCH 03/78] Use master branch for travis badge Co-authored-by: Marco Geweke --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 47e5b37f2..4e2a50e64 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Collection of independent libraries on top of Spring Boot to provide a faster se ## Status -[![build](https://travis-ci.org/otto-de/edison-microservice.svg)](https://travis-ci.org/otto-de/edison-microservice) +[![build](https://travis-ci.org/otto-de/edison-microservice.svg?branch=master)](https://travis-ci.org/otto-de/edison-microservice) [![codecov](https://codecov.io/gh/otto-de/edison-microservice/branch/master/graph/badge.svg)](https://codecov.io/gh/otto-de/edison-microservice) [![Known Vulnerabilities](https://snyk.io/test/github/otto-de/edison-microservice/badge.svg)](https://snyk.io/test/github/otto-de/edison-microservice) [![release](https://maven-badges.herokuapp.com/maven-central/de.otto.edison/edison-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/de.otto.edison/edison-core) From 94888ef0753ca6b2cb5e3429ae356002703ec068 Mon Sep 17 00:00:00 2001 From: Florian Torkler Date: Wed, 28 Aug 2019 14:25:20 +0200 Subject: [PATCH 04/78] make error profile for edison validation configurable via property 'edison.validation.error-profile' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Geißendörfer --- .../web/ErrorHalRepresentation.java | 16 ++++++------ .../web/ErrorHalRepresentationFactory.java | 16 +++++++++++- .../ErrorHalRepresentationFactoryTest.java | 25 +++++++++++++++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java index 90ec11531..35c587a48 100644 --- a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java +++ b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java @@ -5,11 +5,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import de.otto.edison.hal.HalRepresentation; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; import static de.otto.edison.hal.Link.profile; @@ -18,14 +14,12 @@ @JsonDeserialize(builder = ErrorHalRepresentation.Builder.class) public class ErrorHalRepresentation extends HalRepresentation { - private static final String PROFILE_ERROR = "http://spec.otto.de/profiles/error"; - private final String errorMessage; private final Map>> errors; private ErrorHalRepresentation(Builder builder) { super(linkingTo() - .array(profile(PROFILE_ERROR)) + .array(profile(builder.profile)) .build() ); this.errors = builder.errors; @@ -74,6 +68,7 @@ public String getErrorMessage() { public static final class Builder { private Map>> errors = new HashMap<>(); private String errorMessage; + private String profile = ""; private Builder() { } @@ -82,6 +77,11 @@ public ErrorHalRepresentation build() { return new ErrorHalRepresentation(this); } + public Builder withProfile(String profile) { + this.profile = profile; + return this; + } + public Builder withErrors(Map>> errors) { this.errors = errors; return this; diff --git a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java index 239a1cb8e..691d6b1b3 100644 --- a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java +++ b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.support.ResourceBundleMessageSource; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; @@ -17,15 +18,28 @@ public class ErrorHalRepresentationFactory { private final ResourceBundleMessageSource messageSource; private final ObjectMapper objectMapper; + private final String errorProfile; @Autowired - public ErrorHalRepresentationFactory(ResourceBundleMessageSource edisonValidationMessageSource, ObjectMapper objectMapper) { + public ErrorHalRepresentationFactory( + ResourceBundleMessageSource edisonValidationMessageSource, + ObjectMapper objectMapper, + @Value("${edison.validation.error-profile:http://spec.otto.de/profiles/error}") String errorProfile) { this.messageSource = edisonValidationMessageSource; this.objectMapper = objectMapper; + this.errorProfile = errorProfile; + } + + public ErrorHalRepresentation halRepresentationForErrorMessage(String errorMessage) { + return ErrorHalRepresentation.builder() + .withProfile(errorProfile) + .withErrorMessage(errorMessage) + .build(); } public ErrorHalRepresentation halRepresentationForValidationErrors(Errors validationResult) { ErrorHalRepresentation.Builder builder = ErrorHalRepresentation.builder() + .withProfile(errorProfile) .withErrorMessage(String.format("Validation failed. %d error(s)", validationResult.getErrorCount())); validationResult.getAllErrors() diff --git a/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java b/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java index dee78fa09..571542885 100644 --- a/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java +++ b/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java @@ -31,10 +31,27 @@ public void setUp() { } + @Test + public void shouldBuildRepresentationForErrorMessage() { + // given + String someErrorProfile = "someErrorProfile"; + String someErrorMessage = "someErrorMessage"; + final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper(), someErrorProfile); + + // when + ErrorHalRepresentation errorHalRepresentation = factory.halRepresentationForErrorMessage(someErrorMessage); + + // then + assertThat(errorHalRepresentation.getErrors().isEmpty(), is(true)); + assertThat(errorHalRepresentation.getErrorMessage(), is(someErrorMessage)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").isPresent(), is(true)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").get().getHref(), is(someErrorProfile)); + } + @Test public void shouldBuildRepresentationForValidationResults() { // given - final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper()); + final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper(), "someErrorProfile"); // when final Errors mockErrors = mock(Errors.class); @@ -51,6 +68,8 @@ public void shouldBuildRepresentationForValidationResults() { // then assertThat(errorHalRepresentation.getErrorMessage(), is("Validation failed. 1 error(s)")); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").isPresent(), is(true)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").get().getHref(), is("someErrorProfile")); final List> listOfViolations = errorHalRepresentation.getErrors().get("xyzField"); assertThat(listOfViolations, hasSize(1)); assertThat(listOfViolations.get(0), hasEntry("key", "text.not_empty")); @@ -61,7 +80,7 @@ public void shouldBuildRepresentationForValidationResults() { @Test public void shouldNotCrashOnNullValues() { // given - final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper()); + final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper(), "someErrorProfile"); // when final Errors mockErrors = mock(Errors.class); @@ -78,6 +97,8 @@ public void shouldNotCrashOnNullValues() { // then assertThat(errorHalRepresentation.getErrorMessage(), is("Validation failed. 1 error(s)")); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").isPresent(), is(true)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").get().getHref(), is("someErrorProfile")); final List> listOfViolations = errorHalRepresentation.getErrors().get("xyzField"); assertThat(listOfViolations, hasSize(1)); assertThat(listOfViolations.get(0), hasEntry("key", "text.not_empty")); From 5f19e0ff8993aadb3dde0a04c01fc6e6dbaf3d05 Mon Sep 17 00:00:00 2001 From: Florian Torkler Date: Wed, 28 Aug 2019 14:29:35 +0200 Subject: [PATCH 05/78] release 2.0.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Geißendörfer --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 63ef9e566..b815b7cb0 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.0.1-SNAPSHOT' + version = '2.0.1' group = 'de.otto.edison' repositories { From bfcc6659926acff58a0c03b2114117be5dd317d2 Mon Sep 17 00:00:00 2001 From: Florian Torkler Date: Wed, 28 Aug 2019 14:35:19 +0200 Subject: [PATCH 06/78] release 2.0.1 with adjusted CHANGELOG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Geißendörfer --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index be58467a1..cbc34e311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release Notes + +## 2.0.1 +* **[edison-validation]**: Make error profile configurable via application property 'edison.validation.error-profile' + ## 2.0.0 * **[edison-validation]**: Add EnumListValidator to be able to validate a list of enums From c897a1ed768432bfffdc689ed5b1dbe032338bdb Mon Sep 17 00:00:00 2001 From: Florian Torkler Date: Thu, 29 Aug 2019 07:22:55 +0200 Subject: [PATCH 07/78] Next snapshot --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b815b7cb0..547263ec8 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.0.1' + version = '2.0.2-SNAPSHOT' group = 'de.otto.edison' repositories { From daf61c5a62b18f33cd1c34079a5b88107f81c805 Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Mon, 2 Sep 2019 16:56:17 +0200 Subject: [PATCH 08/78] add kotlin togglz support for enum classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- edison-togglz/build.gradle | 20 ++++++ .../de.otto.edison.togglz/EnumAnnotations.kt | 17 +++++ .../EnumClassFeatureProvider.kt | 20 ++++++ .../de.otto.edison.togglz/FeatureEnum.kt | 8 +++ .../FeatureEnumMetaData.kt | 36 ++++++++++ .../KFeatureManagerProvider.kt | 21 ++++++ .../KFeatureManagerSupport.kt | 60 ++++++++++++++++ .../togglz/KFeatureManagerSupportTest.kt | 68 +++++++++++++++++++ .../otto/edison/togglz/KotlinTestFeatures.kt | 26 +++++++ ...org.togglz.core.spi.FeatureManagerProvider | 1 + gradle/dependencies.gradle | 4 +- 11 files changed, 280 insertions(+), 1 deletion(-) create mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt create mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt create mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt create mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt create mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt create mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt create mode 100644 edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt create mode 100644 edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt create mode 100644 edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider diff --git a/edison-togglz/build.gradle b/edison-togglz/build.gradle index 913fc22f1..ac60f7c2a 100644 --- a/edison-togglz/build.gradle +++ b/edison-togglz/build.gradle @@ -1,3 +1,6 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.3.50' +} dependencies { compile project(":edison-core") compile libraries.togglz_console @@ -20,6 +23,10 @@ dependencies { testCompile test_libraries.hamcrest_core testCompile test_libraries.hamcrest_library testCompile test_libraries.spring_boot_starter_test + testCompile test_libraries.mockk + testCompile test_libraries.kotlintest + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" + implementation "org.jetbrains.kotlin:kotlin-reflect" } @@ -71,3 +78,16 @@ uploadArchives { } } } +repositories { + mavenCentral() +} +compileKotlin { + kotlinOptions { + jvmTarget = "1.8" + } +} +compileTestKotlin { + kotlinOptions { + jvmTarget = "1.8" + } +} diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt new file mode 100644 index 000000000..022636127 --- /dev/null +++ b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt @@ -0,0 +1,17 @@ +package de.otto.edison.togglz + +import org.togglz.core.annotation.EnabledByDefault +import org.togglz.core.annotation.Label + +object EnumAnnotations { + + fun getLabel(featureEnum: Enum<*>) = getAnnotation(featureEnum, Label::class.java)?.value ?: featureEnum.name + + fun isEnabledByDefault(featureEnum: Enum<*>) = getAnnotation(featureEnum, EnabledByDefault::class.java) != null + + fun getAnnotation(featureEnum: Enum<*>, annotationClass: Class): A? = + featureEnum.javaClass.getField(featureEnum.name).getAnnotation(annotationClass) + ?: featureEnum.javaClass.getAnnotation(annotationClass) + + fun getAnnotations(featureEnum: Enum<*>) = featureEnum::name.annotations.union(featureEnum::class.annotations) +} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt new file mode 100644 index 000000000..9493cb0b0 --- /dev/null +++ b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt @@ -0,0 +1,20 @@ +package de.otto.edison.togglz + +import org.togglz.core.Feature +import org.togglz.core.spi.FeatureProvider +import java.util.* +import java.util.Collections.unmodifiableSet + +class EnumClassFeatureProvider(featureClass: Class>) : FeatureProvider { + + private val features = featureClass.enumConstants.map { it to FeatureEnum(it) }.toMap() + private val metaData = featureClass.enumConstants + .map { + it.name to FeatureEnumMetaData(it, FeatureEnum(it)) + } + .toMap() + + override fun getFeatures(): Set = unmodifiableSet(HashSet(features.values)) + + override fun getMetaData(feature: Feature) = metaData[feature.name()] +} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt new file mode 100644 index 000000000..3ab403a09 --- /dev/null +++ b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt @@ -0,0 +1,8 @@ +package de.otto.edison.togglz + +import org.togglz.core.Feature + +data class FeatureEnum(private val featureEnum: Enum<*>) : Feature { + + override fun name() = featureEnum.name +} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt new file mode 100644 index 000000000..23cc17ce8 --- /dev/null +++ b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt @@ -0,0 +1,36 @@ +package de.otto.edison.togglz + +import org.togglz.core.Feature +import org.togglz.core.annotation.DefaultActivationStrategy +import org.togglz.core.metadata.FeatureMetaData +import org.togglz.core.metadata.enums.AnnotationFeatureGroup +import org.togglz.core.repository.FeatureState +import org.togglz.core.util.FeatureAnnotations +import java.util.Collections.unmodifiableMap + +class FeatureEnumMetaData(featureEnum: Enum<*>, feature: Feature) : FeatureMetaData { + + private val label = EnumAnnotations.getLabel(featureEnum) + private val defaultFeatureState = FeatureState(feature, EnumAnnotations.isEnabledByDefault(featureEnum)) + private val groups = EnumAnnotations.getAnnotations(featureEnum).map { AnnotationFeatureGroup.build(it.annotationClass.java) } + .toSet() + private val attributes = EnumAnnotations.getAnnotations(featureEnum).map { FeatureAnnotations.getFeatureAttribute(it) } + .map { it[0] to it[1] }.toMap() + + init { + EnumAnnotations.getAnnotation(featureEnum, DefaultActivationStrategy::class.java)?.let { + defaultFeatureState.strategyId = it.id + for (parameter in it.parameters) { + defaultFeatureState.setParameter(parameter.name, parameter.value) + } + } + } + + override fun getLabel() = label + + override fun getDefaultFeatureState(): FeatureState = defaultFeatureState.copy() + + override fun getGroups() = groups + + override fun getAttributes(): Map = unmodifiableMap(attributes) +} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt new file mode 100644 index 000000000..eddd33dd0 --- /dev/null +++ b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt @@ -0,0 +1,21 @@ +package de.otto.edison.togglz + +import org.togglz.core.manager.FeatureManager +import org.togglz.core.spi.FeatureManagerProvider + +class KFeatureManagerProvider: FeatureManagerProvider { + + override fun getFeatureManager(): FeatureManager? { + return instance + } + + override fun priority(): Int { + return 8 + } + + companion object { + var instance: FeatureManager? = null + + } + +} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt new file mode 100644 index 000000000..0dfa2d375 --- /dev/null +++ b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt @@ -0,0 +1,60 @@ +package de.otto.edison.togglz + +import org.togglz.core.Feature +import org.togglz.core.context.FeatureContext.clearCache +import org.togglz.core.context.FeatureContext.getFeatureManager +import org.togglz.core.manager.FeatureManager +import org.togglz.core.manager.FeatureManagerBuilder +import org.togglz.core.repository.FeatureState +import kotlin.reflect.KClass + +object KFeatureManagerSupport { + + fun allEnabledFeatureConfig(featureClass: KClass>) { + val featureManager = initFeatureManager(featureClass) + enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureManager) + clearCache() + } + + fun initFeatureManager(featureClass: KClass>): FeatureManager { + val featureManager = createFeatureManager(featureClass) + KFeatureManagerProvider.instance = featureManager + return featureManager + } + + fun allDisabledFeatureConfig(featureClass: KClass>) { + val featureManager = initFeatureManager(featureClass) + for (feature in featureManager.features) { + featureManager.setFeatureState(FeatureState(feature, false)) + } + clearCache() + } + + private fun createFeatureManager(featureClass: KClass>): FeatureManager { + return FeatureManagerBuilder.begin() + .featureProvider(EnumClassFeatureProvider(featureClass.java)) + .build() + } + + fun enable(feature: Feature) { + getFeatureManager().setFeatureState(FeatureState(feature, true)) + } + + fun disable(feature: Feature) { + getFeatureManager().setFeatureState(FeatureState(feature, false)) + } + + private fun enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureManager: FeatureManager) { + for (feature in featureManager.features) { + if (shouldRunInTests(feature, featureManager)) { + featureManager.setFeatureState(FeatureState(feature, true)) + } + } + } + + fun shouldRunInTests(feature: Feature, featureManager: FeatureManager): Boolean { + val label = featureManager.getMetaData(feature).label + return !label.contains("[inactiveInTests]") + } + +} diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt new file mode 100644 index 000000000..1f5576ef3 --- /dev/null +++ b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt @@ -0,0 +1,68 @@ +package de.otto.edison.togglz + +import io.kotlintest.shouldBe +import io.mockk.junit5.MockKExtension +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtendWith +import org.togglz.core.Feature + +@ExtendWith(MockKExtension::class) +internal class KFeatureManagerSupportTest { + + @BeforeEach + internal fun setUp() { + KFeatureManagerSupport.initFeatureManager(KotlinTestFeatures::class) + } + + @Test + internal fun `should change toggle state after enable`() { + KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) + + KotlinTestFeatures.BAR.isActive() shouldBe true + KotlinTestFeatures.FOO.isActive() shouldBe true + + KFeatureManagerSupport.disable(Feature { KotlinTestFeatures.BAR.name }) + + KotlinTestFeatures.BAR.isActive() shouldBe false + + KFeatureManagerSupport.enable(Feature { KotlinTestFeatures.BAR.name }) + KotlinTestFeatures.BAR.isActive() shouldBe true + } + + @Test + internal fun `should change toggle state after disable`() { + KFeatureManagerSupport.disable(Feature { KotlinTestFeatures.BAR.name }) + + KotlinTestFeatures.BAR.isActive() shouldBe false + + KFeatureManagerSupport.enable(Feature { KotlinTestFeatures.BAR.name }) + KotlinTestFeatures.BAR.isActive() shouldBe true + } + + @Test + internal fun `should enable all toggles`() { + //given + KFeatureManagerSupport.disable(Feature { KotlinTestFeatures.FOO.name }) + KotlinTestFeatures.FOO.isActive() shouldBe false + + //when + KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) + + //then + KotlinTestFeatures.values().forEach { it.isActive() shouldBe true } + } + + @Test + internal fun `should disable all toggles`() { + //given + KFeatureManagerSupport.enable(Feature { KotlinTestFeatures.FOO.name }) + KotlinTestFeatures.FOO.isActive() shouldBe true + + //when + KFeatureManagerSupport.allDisabledFeatureConfig(KotlinTestFeatures::class) + + //then + KotlinTestFeatures.values().forEach { it.isActive() shouldBe false } + } +} \ No newline at end of file diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt new file mode 100644 index 000000000..c430e8939 --- /dev/null +++ b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt @@ -0,0 +1,26 @@ +package de.otto.edison.togglz; + +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.togglz.core.annotation.EnabledByDefault; +import org.togglz.core.annotation.Label; +import org.togglz.core.context.FeatureContext; + +enum class KotlinTestFeatures { + @EnabledByDefault + FOO, + + @Label("bar feature") + BAR; + + fun isActive(): Boolean { + return FeatureContext.getFeatureManager().isActive { name } + } +} + +@Configuration +open class FeatureProviderConfiguration { + + @Bean + open fun featureProvider() = EnumClassFeatureProvider(KotlinTestFeatures::class.java) +} diff --git a/edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider b/edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider new file mode 100644 index 000000000..3a990f7ec --- /dev/null +++ b/edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider @@ -0,0 +1 @@ +de.otto.edison.togglz.KFeatureManagerProvider diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 5e75ac344..1f7616ffe 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -92,7 +92,9 @@ ext { embedded_mongo : "de.flapdoodle.embed:de.flapdoodle.embed.mongo:${test_versions.embedded_mongo}", json_path : "com.jayway.jsonpath:json-path:${versions.json_path}", rest_assured : "io.rest-assured:rest-assured:${test_versions.rest_assured}", - testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}" + testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}", + mockk : "io.mockk:mockk:1.9.3", + kotlintest : "io.kotlintest:kotlintest-runner-junit5:3.4.0" ] gradle_plugins = [ From 8f091ac02f540a5516fc264915b4bb8dd394f9c0 Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Mon, 2 Sep 2019 16:58:15 +0200 Subject: [PATCH 09/78] initialize feature manager to be able to use enum class togglz in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .../configuration/TogglzConfiguration.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java index 0a13a24bf..552320705 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java @@ -3,17 +3,22 @@ import de.otto.edison.authentication.Credentials; import de.otto.edison.togglz.DefaultTogglzConfig; import de.otto.edison.togglz.FeatureClassProvider; +import de.otto.edison.togglz.KFeatureManagerProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; import org.togglz.core.Feature; import org.togglz.core.context.StaticFeatureManagerProvider; import org.togglz.core.manager.FeatureManager; +import org.togglz.core.manager.FeatureManagerBuilder; import org.togglz.core.manager.TogglzConfig; import org.togglz.core.repository.StateRepository; import org.togglz.core.repository.cache.CachingStateRepository; +import org.togglz.core.spi.FeatureProvider; import org.togglz.core.user.SimpleFeatureUser; import org.togglz.core.user.UserProvider; import org.togglz.servlet.TogglzFilter; @@ -67,6 +72,7 @@ public TogglzConfig togglzConfig(final StateRepository stateRepository, } @Bean + @ConditionalOnMissingBean(FeatureManager.class) public FeatureManager featureManager(final TogglzConfig togglzConfig) throws Exception { final FeatureManagerFactory featureManagerFactory = new FeatureManagerFactory(); featureManagerFactory.setTogglzConfig(togglzConfig); @@ -75,6 +81,20 @@ public FeatureManager featureManager(final TogglzConfig togglzConfig) throws Exc return featureManager; } + @Bean + @Primary + @Profile("test") + public FeatureManager testFeatureManager(final Optional featureProvider) throws Exception { + + FeatureManagerBuilder featureManagerBuilder = FeatureManagerBuilder.begin(); + + featureProvider.ifPresent(featureManagerBuilder::featureProvider); + + FeatureManager featureManager = featureManagerBuilder.build(); + KFeatureManagerProvider.Companion.setInstance(featureManager); + return featureManager; + } + private enum Features implements Feature { /* no features */ } From 7372c5839e52e5fb346337de120742ee97c89f74 Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Mon, 2 Sep 2019 17:00:02 +0200 Subject: [PATCH 10/78] rename test method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .../togglz/FeatureTogglesControllerAcceptanceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java b/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java index 0c9778a65..95fe0de2b 100644 --- a/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java @@ -38,7 +38,7 @@ public class FeatureTogglesControllerAcceptanceTest { private int port; @Test - public void shouldTogglesAsJson() { + public void shouldReturnTogglesAsJson() { // when ResponseEntity resource = getResource("http://localhost:" + port + "/togglztest/internal/toggles"); From 106f4787ff4e3911757f956e36f31443e58c4384 Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Mon, 2 Sep 2019 17:01:08 +0200 Subject: [PATCH 11/78] get features for representation from enum constants (java) or feature manager (kotlin) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .../FeatureTogglesRepresentation.java | 5 +++- .../FeatureTogglzRepresentationTest.kt | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java index d6799b8e5..e551cce6d 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java @@ -25,7 +25,10 @@ public static FeatureTogglesRepresentation togglzRepresentation(final FeatureCla } private Map buildTogglzState(final Class featureClass) { - final Feature[] features = featureClass.getEnumConstants(); + Feature[] features = featureClass.getEnumConstants(); + if(features == null || features.length == 0) { + features = getFeatureManager().getFeatures().toArray(new Feature[]{}); + } return stream(features) .collect( toMap(Feature::name, this::toFeatureToggleRepresentation) diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt new file mode 100644 index 000000000..fcb163b53 --- /dev/null +++ b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt @@ -0,0 +1,29 @@ +package de.otto.edison.togglz.controller.representation + +import de.otto.edison.togglz.EmptyFeatures +import de.otto.edison.togglz.KFeatureManagerSupport +import de.otto.edison.togglz.KotlinTestFeatures +import de.otto.edison.togglz.controller.FeatureToggleRepresentation +import de.otto.edison.togglz.controller.FeatureTogglesRepresentation.togglzRepresentation +import io.kotlintest.shouldBe +import org.junit.jupiter.api.Test + +internal class FeatureTogglzRepresentationTest { + @Test + fun testGetFeatureRepresentationForKotlinEnumClass() { + + //given + KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) + + //when + val togglzRepresentation = togglzRepresentation { EmptyFeatures::class.java } + + //then + val features = togglzRepresentation.features + val fooRepresentation: FeatureToggleRepresentation? = features["FOO"] + fooRepresentation!!.enabled shouldBe true + + val barRepresentation: FeatureToggleRepresentation? = features["BAR"] + barRepresentation!!.enabled shouldBe true + } +} \ No newline at end of file From 05b79ac6f1c79651c175eb484f355e084999d01c Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Tue, 3 Sep 2019 09:50:44 +0200 Subject: [PATCH 12/78] fix dependending tests and expect kotlin enum features to be of FeatureEnum type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .../controller/FeatureTogglesRepresentation.java | 8 ++++---- .../edison/togglz/DefaultTogglzConfigTest.java | 7 +++++++ .../FeatureTogglesRepresentationTest.java | 8 ++++++++ .../edison/togglz/KFeatureManagerSupportTest.kt | 6 ++++++ ...Test.kt => FeatureTogglesRepresentationTest.kt} | 14 +++++++++----- 5 files changed, 34 insertions(+), 9 deletions(-) rename edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/{FeatureTogglzRepresentationTest.kt => FeatureTogglesRepresentationTest.kt} (72%) diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java index e551cce6d..f6f5103f5 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java @@ -1,6 +1,7 @@ package de.otto.edison.togglz.controller; import de.otto.edison.togglz.FeatureClassProvider; +import de.otto.edison.togglz.FeatureEnum; import net.jcip.annotations.Immutable; import org.togglz.core.Feature; import org.togglz.core.annotation.Label; @@ -25,10 +26,9 @@ public static FeatureTogglesRepresentation togglzRepresentation(final FeatureCla } private Map buildTogglzState(final Class featureClass) { - Feature[] features = featureClass.getEnumConstants(); - if(features == null || features.length == 0) { - features = getFeatureManager().getFeatures().toArray(new Feature[]{}); - } + Feature[] features = !featureClass.equals(FeatureEnum.class) + ? featureClass.getEnumConstants() + :getFeatureManager().getFeatures().toArray(new Feature[]{}); return stream(features) .collect( toMap(Feature::name, this::toFeatureToggleRepresentation) diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java index 85c88326a..0a4915816 100644 --- a/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java @@ -1,5 +1,7 @@ package de.otto.edison.togglz; +import de.otto.edison.testsupport.togglz.FeatureManagerSupport; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -22,6 +24,11 @@ public class DefaultTogglzConfigTest { @Autowired private TogglzConfig togglzConfig; + @BeforeEach + void setUp() { + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + } + @Test public void shouldCreateTogglzConfigBySpring() { assertThat(togglzConfig, is(not(nullValue()))); diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java index cbfc132f0..7c864996d 100644 --- a/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java @@ -1,8 +1,11 @@ package de.otto.edison.togglz.controller; +import de.otto.edison.testsupport.togglz.FeatureManagerSupport; import de.otto.edison.togglz.EmptyFeatures; import de.otto.edison.togglz.TestFeatures; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.togglz.core.manager.FeatureManager; import java.util.Map; @@ -15,6 +18,11 @@ public class FeatureTogglesRepresentationTest { private FeatureTogglesRepresentation testee; + @BeforeEach + void setUp() { + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + } + @Test public void testGetFeatureRepresentation() { testee = togglzRepresentation(() -> TestFeatures.class); diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt index 1f5576ef3..a30cd926b 100644 --- a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt +++ b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt @@ -2,6 +2,7 @@ package de.otto.edison.togglz import io.kotlintest.shouldBe import io.mockk.junit5.MockKExtension +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -65,4 +66,9 @@ internal class KFeatureManagerSupportTest { //then KotlinTestFeatures.values().forEach { it.isActive() shouldBe false } } + + @AfterEach + internal fun tearDown() { + KFeatureManagerProvider.instance = null + } } \ No newline at end of file diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt similarity index 72% rename from edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt rename to edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt index fcb163b53..69e18cc96 100644 --- a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglzRepresentationTest.kt +++ b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt @@ -1,14 +1,13 @@ package de.otto.edison.togglz.controller.representation -import de.otto.edison.togglz.EmptyFeatures -import de.otto.edison.togglz.KFeatureManagerSupport -import de.otto.edison.togglz.KotlinTestFeatures +import de.otto.edison.togglz.* import de.otto.edison.togglz.controller.FeatureToggleRepresentation import de.otto.edison.togglz.controller.FeatureTogglesRepresentation.togglzRepresentation import io.kotlintest.shouldBe +import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Test -internal class FeatureTogglzRepresentationTest { +internal class FeatureTogglesRepresentationTest { @Test fun testGetFeatureRepresentationForKotlinEnumClass() { @@ -16,7 +15,7 @@ internal class FeatureTogglzRepresentationTest { KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) //when - val togglzRepresentation = togglzRepresentation { EmptyFeatures::class.java } + val togglzRepresentation = togglzRepresentation { FeatureEnum::class.java } //then val features = togglzRepresentation.features @@ -26,4 +25,9 @@ internal class FeatureTogglzRepresentationTest { val barRepresentation: FeatureToggleRepresentation? = features["BAR"] barRepresentation!!.enabled shouldBe true } + + @AfterEach + internal fun tearDown() { + KFeatureManagerProvider.instance = null + } } \ No newline at end of file From 320f52cb6ea9a05c179fd5440251158de69dfdae Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Tue, 3 Sep 2019 13:53:37 +0200 Subject: [PATCH 13/78] move test feature manager to test package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .../configuration/TogglzConfiguration.java | 14 --------- .../TestTogglesConfiguration.java | 30 +++++++++++++++++++ ...org.togglz.core.spi.FeatureManagerProvider | 1 + 3 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java create mode 100644 examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java index 552320705..ad897c3a2 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java @@ -81,20 +81,6 @@ public FeatureManager featureManager(final TogglzConfig togglzConfig) throws Exc return featureManager; } - @Bean - @Primary - @Profile("test") - public FeatureManager testFeatureManager(final Optional featureProvider) throws Exception { - - FeatureManagerBuilder featureManagerBuilder = FeatureManagerBuilder.begin(); - - featureProvider.ifPresent(featureManagerBuilder::featureProvider); - - FeatureManager featureManager = featureManagerBuilder.build(); - KFeatureManagerProvider.Companion.setInstance(featureManager); - return featureManager; - } - private enum Features implements Feature { /* no features */ } diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java b/edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java new file mode 100644 index 000000000..3ad31ed1a --- /dev/null +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java @@ -0,0 +1,30 @@ +package de.otto.edison.togglz.configuration; + +import de.otto.edison.togglz.KFeatureManagerProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; +import org.togglz.core.manager.FeatureManager; +import org.togglz.core.manager.FeatureManagerBuilder; +import org.togglz.core.spi.FeatureProvider; + +import java.util.Optional; + +@Configuration +public class TestTogglesConfiguration { + + @Bean + @Primary + @Profile("test") + public FeatureManager testFeatureManager(final Optional featureProvider) throws Exception { + + FeatureManagerBuilder featureManagerBuilder = FeatureManagerBuilder.begin(); + + featureProvider.ifPresent(featureManagerBuilder::featureProvider); + + FeatureManager featureManager = featureManagerBuilder.build(); + KFeatureManagerProvider.Companion.setInstance(featureManager); + return featureManager; + } +} diff --git a/examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider b/examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider new file mode 100644 index 000000000..3a990f7ec --- /dev/null +++ b/examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider @@ -0,0 +1 @@ +de.otto.edison.togglz.KFeatureManagerProvider From c1dafa9da9075a2ae915530eed8eeb917c30f5c4 Mon Sep 17 00:00:00 2001 From: Thimo Tollmien Date: Tue, 3 Sep 2019 13:56:40 +0200 Subject: [PATCH 14/78] add kotlintest to gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ef8550dd6..b87bb0b50 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .gradle .shelf build +.kotlintest/ out **/build *.iws From 9b1a94e1a426b1dc941e225367fa3f6bed10f517 Mon Sep 17 00:00:00 2001 From: Frank Bregulla Date: Tue, 3 Sep 2019 17:28:27 +0200 Subject: [PATCH 15/78] move kotlin togglz wrapper to its own tokklz library and use featureManager in featureTogglzRepresentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Erik Wölfel --- .gitignore | 1 - edison-togglz/build.gradle | 16 +--- .../configuration/TogglzConfiguration.java | 5 -- .../controller/FeatureTogglesController.java | 10 +-- .../FeatureTogglesRepresentation.java | 46 ++++-------- .../de.otto.edison.togglz/EnumAnnotations.kt | 17 ----- .../EnumClassFeatureProvider.kt | 20 ----- .../de.otto.edison.togglz/FeatureEnum.kt | 8 -- .../FeatureEnumMetaData.kt | 36 --------- .../KFeatureManagerProvider.kt | 21 ------ .../KFeatureManagerSupport.kt | 60 --------------- .../TestTogglesConfiguration.java | 30 -------- .../FeatureTogglesRepresentationTest.java | 16 ++-- .../togglz/KFeatureManagerSupportTest.kt | 74 ------------------- .../otto/edison/togglz/KotlinTestFeatures.kt | 26 ------- .../FeatureTogglesRepresentationTest.kt | 33 --------- ...org.togglz.core.spi.FeatureManagerProvider | 1 - ...org.togglz.core.spi.FeatureManagerProvider | 1 - gradle/dependencies.gradle | 5 +- 19 files changed, 28 insertions(+), 398 deletions(-) delete mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt delete mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt delete mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt delete mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt delete mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt delete mode 100644 edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt delete mode 100644 edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java delete mode 100644 edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt delete mode 100644 edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt delete mode 100644 edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt delete mode 100644 edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider delete mode 100644 examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider diff --git a/.gitignore b/.gitignore index b87bb0b50..ef8550dd6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ .gradle .shelf build -.kotlintest/ out **/build *.iws diff --git a/edison-togglz/build.gradle b/edison-togglz/build.gradle index ac60f7c2a..453180895 100644 --- a/edison-togglz/build.gradle +++ b/edison-togglz/build.gradle @@ -23,10 +23,8 @@ dependencies { testCompile test_libraries.hamcrest_core testCompile test_libraries.hamcrest_library testCompile test_libraries.spring_boot_starter_test - testCompile test_libraries.mockk - testCompile test_libraries.kotlintest - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8" - implementation "org.jetbrains.kotlin:kotlin-reflect" + testCompile test_libraries.togglz_testing + } @@ -81,13 +79,3 @@ uploadArchives { repositories { mavenCentral() } -compileKotlin { - kotlinOptions { - jvmTarget = "1.8" - } -} -compileTestKotlin { - kotlinOptions { - jvmTarget = "1.8" - } -} diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java index ad897c3a2..099d68150 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java @@ -3,22 +3,17 @@ import de.otto.edison.authentication.Credentials; import de.otto.edison.togglz.DefaultTogglzConfig; import de.otto.edison.togglz.FeatureClassProvider; -import de.otto.edison.togglz.KFeatureManagerProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; import org.togglz.core.Feature; import org.togglz.core.context.StaticFeatureManagerProvider; import org.togglz.core.manager.FeatureManager; -import org.togglz.core.manager.FeatureManagerBuilder; import org.togglz.core.manager.TogglzConfig; import org.togglz.core.repository.StateRepository; import org.togglz.core.repository.cache.CachingStateRepository; -import org.togglz.core.spi.FeatureProvider; import org.togglz.core.user.SimpleFeatureUser; import org.togglz.core.user.UserProvider; import org.togglz.servlet.TogglzFilter; diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java index 3ea5a1de4..76e79ce9b 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java @@ -1,9 +1,9 @@ package de.otto.edison.togglz.controller; -import de.otto.edison.togglz.FeatureClassProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.togglz.core.manager.FeatureManager; import static de.otto.edison.togglz.controller.FeatureTogglesRepresentation.togglzRepresentation; import static org.springframework.web.bind.annotation.RequestMethod.GET; @@ -11,11 +11,11 @@ @RestController public class FeatureTogglesController { - private final FeatureClassProvider featureClassProvider; + private final FeatureManager featureManager; @Autowired - public FeatureTogglesController(final FeatureClassProvider featureClassProvider) { - this.featureClassProvider = featureClassProvider; + public FeatureTogglesController(final FeatureManager featureManager) { + this.featureManager = featureManager; } @RequestMapping( @@ -26,6 +26,6 @@ public FeatureTogglesController(final FeatureClassProvider featureClassProvider) method = GET ) public FeatureTogglesRepresentation getStatusAsJson() { - return togglzRepresentation(featureClassProvider); + return togglzRepresentation(featureManager); } } diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java index f6f5103f5..40ad55c56 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java @@ -1,61 +1,41 @@ package de.otto.edison.togglz.controller; -import de.otto.edison.togglz.FeatureClassProvider; -import de.otto.edison.togglz.FeatureEnum; import net.jcip.annotations.Immutable; import org.togglz.core.Feature; -import org.togglz.core.annotation.Label; +import org.togglz.core.manager.FeatureManager; import java.util.Map; import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; -import static org.togglz.core.context.FeatureContext.getFeatureManager; @Immutable public class FeatureTogglesRepresentation { public final Map features; - private FeatureTogglesRepresentation(final Class featureClass) { - this.features = buildTogglzState(featureClass); + private FeatureTogglesRepresentation(final FeatureManager featureManager) { + this.features = buildTogglzState(featureManager); } - public static FeatureTogglesRepresentation togglzRepresentation(final FeatureClassProvider featureClassProvider) { - return new FeatureTogglesRepresentation(featureClassProvider.getFeatureClass()); + public static FeatureTogglesRepresentation togglzRepresentation(final FeatureManager featureManager) { + return new FeatureTogglesRepresentation(featureManager); } - private Map buildTogglzState(final Class featureClass) { - Feature[] features = !featureClass.equals(FeatureEnum.class) - ? featureClass.getEnumConstants() - :getFeatureManager().getFeatures().toArray(new Feature[]{}); + private Map buildTogglzState(final FeatureManager featureManager) { + Feature[] features = featureManager.getFeatures().toArray(new Feature[]{}); return stream(features) .collect( - toMap(Feature::name, this::toFeatureToggleRepresentation) + toMap(Feature::name, feature -> toFeatureToggleRepresentation(feature, featureManager)) ); } - private FeatureToggleRepresentation toFeatureToggleRepresentation(final Feature feature) { - final Label label = getLabelAnnotation(feature); + private FeatureToggleRepresentation toFeatureToggleRepresentation(final Feature feature, FeatureManager featureManager) { + + final String label = featureManager.getMetaData(feature).getLabel(); return new FeatureToggleRepresentation( - label != null ? label.value() : feature.name(), - getFeatureManager().getFeatureState(feature).isEnabled(), + label != null ? label : feature.name(), + featureManager.getFeatureState(feature).isEnabled(), null); } - - public static Label getLabelAnnotation(Feature feature) { - try { - Class featureClass = feature.getClass(); - Label fieldAnnotation = featureClass.getField(feature.name()).getAnnotation(Label.class); - Label classAnnotation = featureClass.getAnnotation(Label.class); - - return fieldAnnotation != null ? fieldAnnotation : classAnnotation; - } catch (SecurityException e) { - // ignore - } catch (NoSuchFieldException e) { - // ignore - } - return null; - } - } diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt deleted file mode 100644 index 022636127..000000000 --- a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumAnnotations.kt +++ /dev/null @@ -1,17 +0,0 @@ -package de.otto.edison.togglz - -import org.togglz.core.annotation.EnabledByDefault -import org.togglz.core.annotation.Label - -object EnumAnnotations { - - fun getLabel(featureEnum: Enum<*>) = getAnnotation(featureEnum, Label::class.java)?.value ?: featureEnum.name - - fun isEnabledByDefault(featureEnum: Enum<*>) = getAnnotation(featureEnum, EnabledByDefault::class.java) != null - - fun getAnnotation(featureEnum: Enum<*>, annotationClass: Class): A? = - featureEnum.javaClass.getField(featureEnum.name).getAnnotation(annotationClass) - ?: featureEnum.javaClass.getAnnotation(annotationClass) - - fun getAnnotations(featureEnum: Enum<*>) = featureEnum::name.annotations.union(featureEnum::class.annotations) -} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt deleted file mode 100644 index 9493cb0b0..000000000 --- a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/EnumClassFeatureProvider.kt +++ /dev/null @@ -1,20 +0,0 @@ -package de.otto.edison.togglz - -import org.togglz.core.Feature -import org.togglz.core.spi.FeatureProvider -import java.util.* -import java.util.Collections.unmodifiableSet - -class EnumClassFeatureProvider(featureClass: Class>) : FeatureProvider { - - private val features = featureClass.enumConstants.map { it to FeatureEnum(it) }.toMap() - private val metaData = featureClass.enumConstants - .map { - it.name to FeatureEnumMetaData(it, FeatureEnum(it)) - } - .toMap() - - override fun getFeatures(): Set = unmodifiableSet(HashSet(features.values)) - - override fun getMetaData(feature: Feature) = metaData[feature.name()] -} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt deleted file mode 100644 index 3ab403a09..000000000 --- a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnum.kt +++ /dev/null @@ -1,8 +0,0 @@ -package de.otto.edison.togglz - -import org.togglz.core.Feature - -data class FeatureEnum(private val featureEnum: Enum<*>) : Feature { - - override fun name() = featureEnum.name -} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt deleted file mode 100644 index 23cc17ce8..000000000 --- a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/FeatureEnumMetaData.kt +++ /dev/null @@ -1,36 +0,0 @@ -package de.otto.edison.togglz - -import org.togglz.core.Feature -import org.togglz.core.annotation.DefaultActivationStrategy -import org.togglz.core.metadata.FeatureMetaData -import org.togglz.core.metadata.enums.AnnotationFeatureGroup -import org.togglz.core.repository.FeatureState -import org.togglz.core.util.FeatureAnnotations -import java.util.Collections.unmodifiableMap - -class FeatureEnumMetaData(featureEnum: Enum<*>, feature: Feature) : FeatureMetaData { - - private val label = EnumAnnotations.getLabel(featureEnum) - private val defaultFeatureState = FeatureState(feature, EnumAnnotations.isEnabledByDefault(featureEnum)) - private val groups = EnumAnnotations.getAnnotations(featureEnum).map { AnnotationFeatureGroup.build(it.annotationClass.java) } - .toSet() - private val attributes = EnumAnnotations.getAnnotations(featureEnum).map { FeatureAnnotations.getFeatureAttribute(it) } - .map { it[0] to it[1] }.toMap() - - init { - EnumAnnotations.getAnnotation(featureEnum, DefaultActivationStrategy::class.java)?.let { - defaultFeatureState.strategyId = it.id - for (parameter in it.parameters) { - defaultFeatureState.setParameter(parameter.name, parameter.value) - } - } - } - - override fun getLabel() = label - - override fun getDefaultFeatureState(): FeatureState = defaultFeatureState.copy() - - override fun getGroups() = groups - - override fun getAttributes(): Map = unmodifiableMap(attributes) -} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt deleted file mode 100644 index eddd33dd0..000000000 --- a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerProvider.kt +++ /dev/null @@ -1,21 +0,0 @@ -package de.otto.edison.togglz - -import org.togglz.core.manager.FeatureManager -import org.togglz.core.spi.FeatureManagerProvider - -class KFeatureManagerProvider: FeatureManagerProvider { - - override fun getFeatureManager(): FeatureManager? { - return instance - } - - override fun priority(): Int { - return 8 - } - - companion object { - var instance: FeatureManager? = null - - } - -} \ No newline at end of file diff --git a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt b/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt deleted file mode 100644 index 0dfa2d375..000000000 --- a/edison-togglz/src/main/kotlin/de.otto.edison.togglz/KFeatureManagerSupport.kt +++ /dev/null @@ -1,60 +0,0 @@ -package de.otto.edison.togglz - -import org.togglz.core.Feature -import org.togglz.core.context.FeatureContext.clearCache -import org.togglz.core.context.FeatureContext.getFeatureManager -import org.togglz.core.manager.FeatureManager -import org.togglz.core.manager.FeatureManagerBuilder -import org.togglz.core.repository.FeatureState -import kotlin.reflect.KClass - -object KFeatureManagerSupport { - - fun allEnabledFeatureConfig(featureClass: KClass>) { - val featureManager = initFeatureManager(featureClass) - enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureManager) - clearCache() - } - - fun initFeatureManager(featureClass: KClass>): FeatureManager { - val featureManager = createFeatureManager(featureClass) - KFeatureManagerProvider.instance = featureManager - return featureManager - } - - fun allDisabledFeatureConfig(featureClass: KClass>) { - val featureManager = initFeatureManager(featureClass) - for (feature in featureManager.features) { - featureManager.setFeatureState(FeatureState(feature, false)) - } - clearCache() - } - - private fun createFeatureManager(featureClass: KClass>): FeatureManager { - return FeatureManagerBuilder.begin() - .featureProvider(EnumClassFeatureProvider(featureClass.java)) - .build() - } - - fun enable(feature: Feature) { - getFeatureManager().setFeatureState(FeatureState(feature, true)) - } - - fun disable(feature: Feature) { - getFeatureManager().setFeatureState(FeatureState(feature, false)) - } - - private fun enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureManager: FeatureManager) { - for (feature in featureManager.features) { - if (shouldRunInTests(feature, featureManager)) { - featureManager.setFeatureState(FeatureState(feature, true)) - } - } - } - - fun shouldRunInTests(feature: Feature, featureManager: FeatureManager): Boolean { - val label = featureManager.getMetaData(feature).label - return !label.contains("[inactiveInTests]") - } - -} diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java b/edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java deleted file mode 100644 index 3ad31ed1a..000000000 --- a/edison-togglz/src/test/java/de/otto/edison/togglz/configuration/TestTogglesConfiguration.java +++ /dev/null @@ -1,30 +0,0 @@ -package de.otto.edison.togglz.configuration; - -import de.otto.edison.togglz.KFeatureManagerProvider; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; -import org.togglz.core.manager.FeatureManager; -import org.togglz.core.manager.FeatureManagerBuilder; -import org.togglz.core.spi.FeatureProvider; - -import java.util.Optional; - -@Configuration -public class TestTogglesConfiguration { - - @Bean - @Primary - @Profile("test") - public FeatureManager testFeatureManager(final Optional featureProvider) throws Exception { - - FeatureManagerBuilder featureManagerBuilder = FeatureManagerBuilder.begin(); - - featureProvider.ifPresent(featureManagerBuilder::featureProvider); - - FeatureManager featureManager = featureManagerBuilder.build(); - KFeatureManagerProvider.Companion.setInstance(featureManager); - return featureManager; - } -} diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java index 7c864996d..1f81b2921 100644 --- a/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java @@ -1,11 +1,10 @@ package de.otto.edison.togglz.controller; -import de.otto.edison.testsupport.togglz.FeatureManagerSupport; import de.otto.edison.togglz.EmptyFeatures; import de.otto.edison.togglz.TestFeatures; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.togglz.core.manager.FeatureManager; +import org.togglz.testing.TestFeatureManager; import java.util.Map; @@ -18,22 +17,19 @@ public class FeatureTogglesRepresentationTest { private FeatureTogglesRepresentation testee; - @BeforeEach - void setUp() { - FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); - } - @Test public void testGetFeatureRepresentation() { - testee = togglzRepresentation(() -> TestFeatures.class); + FeatureManager featureManager = new TestFeatureManager(TestFeatures.class); + testee = togglzRepresentation(featureManager); final Map features = testee.features; - assertThat(features.get("TEST_FEATURE"), is(new FeatureToggleRepresentation("a test feature toggle", true, null))); + assertThat(features.get("TEST_FEATURE"), is(new FeatureToggleRepresentation("a test feature toggle", false, null))); } @Test public void testGetEmptyFeatureRepresentation() { - testee = togglzRepresentation(() -> EmptyFeatures.class); + FeatureManager featureManager = new TestFeatureManager(EmptyFeatures.class); + testee = togglzRepresentation(featureManager); final Map features = testee.features; assertThat(features, is(notNullValue())); diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt deleted file mode 100644 index a30cd926b..000000000 --- a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KFeatureManagerSupportTest.kt +++ /dev/null @@ -1,74 +0,0 @@ -package de.otto.edison.togglz - -import io.kotlintest.shouldBe -import io.mockk.junit5.MockKExtension -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.extension.ExtendWith -import org.togglz.core.Feature - -@ExtendWith(MockKExtension::class) -internal class KFeatureManagerSupportTest { - - @BeforeEach - internal fun setUp() { - KFeatureManagerSupport.initFeatureManager(KotlinTestFeatures::class) - } - - @Test - internal fun `should change toggle state after enable`() { - KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) - - KotlinTestFeatures.BAR.isActive() shouldBe true - KotlinTestFeatures.FOO.isActive() shouldBe true - - KFeatureManagerSupport.disable(Feature { KotlinTestFeatures.BAR.name }) - - KotlinTestFeatures.BAR.isActive() shouldBe false - - KFeatureManagerSupport.enable(Feature { KotlinTestFeatures.BAR.name }) - KotlinTestFeatures.BAR.isActive() shouldBe true - } - - @Test - internal fun `should change toggle state after disable`() { - KFeatureManagerSupport.disable(Feature { KotlinTestFeatures.BAR.name }) - - KotlinTestFeatures.BAR.isActive() shouldBe false - - KFeatureManagerSupport.enable(Feature { KotlinTestFeatures.BAR.name }) - KotlinTestFeatures.BAR.isActive() shouldBe true - } - - @Test - internal fun `should enable all toggles`() { - //given - KFeatureManagerSupport.disable(Feature { KotlinTestFeatures.FOO.name }) - KotlinTestFeatures.FOO.isActive() shouldBe false - - //when - KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) - - //then - KotlinTestFeatures.values().forEach { it.isActive() shouldBe true } - } - - @Test - internal fun `should disable all toggles`() { - //given - KFeatureManagerSupport.enable(Feature { KotlinTestFeatures.FOO.name }) - KotlinTestFeatures.FOO.isActive() shouldBe true - - //when - KFeatureManagerSupport.allDisabledFeatureConfig(KotlinTestFeatures::class) - - //then - KotlinTestFeatures.values().forEach { it.isActive() shouldBe false } - } - - @AfterEach - internal fun tearDown() { - KFeatureManagerProvider.instance = null - } -} \ No newline at end of file diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt deleted file mode 100644 index c430e8939..000000000 --- a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/KotlinTestFeatures.kt +++ /dev/null @@ -1,26 +0,0 @@ -package de.otto.edison.togglz; - -import org.springframework.context.annotation.Bean -import org.springframework.context.annotation.Configuration -import org.togglz.core.annotation.EnabledByDefault; -import org.togglz.core.annotation.Label; -import org.togglz.core.context.FeatureContext; - -enum class KotlinTestFeatures { - @EnabledByDefault - FOO, - - @Label("bar feature") - BAR; - - fun isActive(): Boolean { - return FeatureContext.getFeatureManager().isActive { name } - } -} - -@Configuration -open class FeatureProviderConfiguration { - - @Bean - open fun featureProvider() = EnumClassFeatureProvider(KotlinTestFeatures::class.java) -} diff --git a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt b/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt deleted file mode 100644 index 69e18cc96..000000000 --- a/edison-togglz/src/test/kotlin/de/otto/edison/togglz/controller/representation/FeatureTogglesRepresentationTest.kt +++ /dev/null @@ -1,33 +0,0 @@ -package de.otto.edison.togglz.controller.representation - -import de.otto.edison.togglz.* -import de.otto.edison.togglz.controller.FeatureToggleRepresentation -import de.otto.edison.togglz.controller.FeatureTogglesRepresentation.togglzRepresentation -import io.kotlintest.shouldBe -import org.junit.jupiter.api.AfterEach -import org.junit.jupiter.api.Test - -internal class FeatureTogglesRepresentationTest { - @Test - fun testGetFeatureRepresentationForKotlinEnumClass() { - - //given - KFeatureManagerSupport.allEnabledFeatureConfig(KotlinTestFeatures::class) - - //when - val togglzRepresentation = togglzRepresentation { FeatureEnum::class.java } - - //then - val features = togglzRepresentation.features - val fooRepresentation: FeatureToggleRepresentation? = features["FOO"] - fooRepresentation!!.enabled shouldBe true - - val barRepresentation: FeatureToggleRepresentation? = features["BAR"] - barRepresentation!!.enabled shouldBe true - } - - @AfterEach - internal fun tearDown() { - KFeatureManagerProvider.instance = null - } -} \ No newline at end of file diff --git a/edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider b/edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider deleted file mode 100644 index 3a990f7ec..000000000 --- a/edison-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider +++ /dev/null @@ -1 +0,0 @@ -de.otto.edison.togglz.KFeatureManagerProvider diff --git a/examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider b/examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider deleted file mode 100644 index 3a990f7ec..000000000 --- a/examples/example-togglz/src/test/resources/META-INF/services/org.togglz.core.spi.FeatureManagerProvider +++ /dev/null @@ -1 +0,0 @@ -de.otto.edison.togglz.KFeatureManagerProvider diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 1f7616ffe..b19b27a65 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -65,6 +65,7 @@ ext { togglz_console : "org.togglz:togglz-console:${versions.togglz}", togglz_spring_web : "org.togglz:togglz-spring-web:${versions.togglz}", togglz_spring_boot_starter : "org.togglz:togglz-spring-boot-starter:${versions.togglz}", + togglz_testing : "org.togglz:togglz-testing:${versions.togglz}", mongodb_driver : "org.mongodb:mongodb-driver:${versions.mongodb_driver}", caffeine : "com.github.ben-manes.caffeine:caffeine:${versions.caffeine}", unboundid_ldapsdk_minimal_edition : "com.unboundid:unboundid-ldapsdk-minimal-edition:${versions.unboundid_ldapsdk_minimal_edition}", @@ -92,9 +93,7 @@ ext { embedded_mongo : "de.flapdoodle.embed:de.flapdoodle.embed.mongo:${test_versions.embedded_mongo}", json_path : "com.jayway.jsonpath:json-path:${versions.json_path}", rest_assured : "io.rest-assured:rest-assured:${test_versions.rest_assured}", - testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}", - mockk : "io.mockk:mockk:1.9.3", - kotlintest : "io.kotlintest:kotlintest-runner-junit5:3.4.0" + testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}" ] gradle_plugins = [ From 0a06f02d46d8954dda3ff1818e255899bf3399f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Wed, 4 Sep 2019 09:10:02 +0200 Subject: [PATCH 16/78] fix acceptance test by using the injected feature manager Co-authored-by: Frank Bregulla --- .../togglz/FeatureManagerSupport.java | 35 +++++++++----- ...eatureTogglesControllerAcceptanceTest.java | 47 +++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java b/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java index 50b7c77bc..99d17da29 100644 --- a/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java +++ b/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java @@ -2,6 +2,7 @@ import org.togglz.core.Feature; import org.togglz.core.context.FeatureContext; +import org.togglz.core.manager.FeatureManager; import org.togglz.core.repository.FeatureState; import org.togglz.core.util.FeatureAnnotations; @@ -12,36 +13,46 @@ public class FeatureManagerSupport { public static void allEnabledFeatureConfig(final Class featureClass) { TestFeatureManager featureManager = new TestFeatureManager(featureClass); - enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureClass,featureManager); TestFeatureManagerProvider.setFeatureManager(featureManager); + allEnabledFeatureConfig(featureManager); + } + + public static void allEnabledFeatureConfig(FeatureManager featureManager) { + enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureManager); clearCache(); } public static void allDisabledFeatureConfig(final Class featureClass) { TestFeatureManager featureManager = new TestFeatureManager(featureClass); - for (Feature feature : featureClass.getEnumConstants()) { - featureManager.disable(feature); - } TestFeatureManagerProvider.setFeatureManager(featureManager); - clearCache(); + allDisabledFeatureConfig(featureManager); } + public static void allDisabledFeatureConfig(FeatureManager featureManager) { + featureManager.getFeatures().forEach(feature -> { + featureManager.setFeatureState(new FeatureState(feature, false)); + }); + clearCache(); + } public static void disable(final Feature feature) { FeatureContext.getFeatureManager().setFeatureState(new FeatureState(feature, false)); } - private static void enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(final Class featureClass, final TestFeatureManager featureManager) { - for (Feature feature : featureClass.getEnumConstants()) { - if (shouldRunInTests(feature)) { - featureManager.enable(feature); + private static void enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(final FeatureManager featureManager) { + featureManager.getFeatures().forEach(feature -> { + if (shouldRunInTests(featureManager, feature)) { + featureManager.setFeatureState(new FeatureState(feature, true)); } - } + }); } public static boolean shouldRunInTests(Feature feature) { - String label = FeatureAnnotations.getLabel(feature); - return !label.contains("[inactiveInTests]"); + return shouldRunInTests(getFeatureManager(), feature); + } + + private static boolean shouldRunInTests(FeatureManager featureManager, Feature feature) { + return !featureManager.getMetaData(feature).getAttributes().containsKey("inactiveInTests"); } public static void enable(final Feature feature) { diff --git a/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java b/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java index 95fe0de2b..a60233240 100644 --- a/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java @@ -2,19 +2,33 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import de.otto.edison.testsupport.togglz.FeatureManagerSupport; +import de.otto.edison.togglz.TestFeatures; import de.otto.edison.togglz.TestServer; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.client.RestTemplate; +import org.togglz.core.Feature; +import org.togglz.core.manager.FeatureManager; +import org.togglz.core.manager.TogglzConfig; +import org.togglz.core.repository.StateRepository; +import org.togglz.core.repository.mem.InMemoryStateRepository; +import org.togglz.core.user.NoOpUserProvider; +import org.togglz.core.user.UserProvider; import java.io.IOException; @@ -26,6 +40,7 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = RANDOM_PORT, classes = {TestServer.class}) +@ContextConfiguration(classes = {FeatureTogglesControllerAcceptanceTest.TogglzConfiguration.class}) @ActiveProfiles("test") public class FeatureTogglesControllerAcceptanceTest { @@ -34,9 +49,17 @@ public class FeatureTogglesControllerAcceptanceTest { @Autowired private ObjectMapper objectMapper; + @Autowired + private FeatureManager featureManager; + @LocalServerPort private int port; + @BeforeEach + void setUp() { + FeatureManagerSupport.allEnabledFeatureConfig(featureManager); + } + @Test public void shouldReturnTogglesAsJson() { // when @@ -68,5 +91,29 @@ JsonNode jsonNode(ResponseEntity resource) { } } + @Configuration + static class TogglzConfiguration{ + + @Bean + @Profile("test") + public TogglzConfig togglzConfig() { + return new TogglzConfig() { + @Override + public Class getFeatureClass() { + return TestFeatures.class; + } + @Override + public StateRepository getStateRepository() { + return new InMemoryStateRepository(); + } + + @Override + public UserProvider getUserProvider() { + return new NoOpUserProvider(); + } + }; + } + + } } From bbd7e7f65f2c13a4f74af4934180cb7d4b952ee6 Mon Sep 17 00:00:00 2001 From: Frank Bregulla Date: Wed, 4 Sep 2019 10:56:45 +0200 Subject: [PATCH 17/78] Fix FeatureManagerSupport and add tests Co-authored-by: Thimo Tollmien --- CHANGELOG.md | 5 +++ build.gradle | 2 +- edison-testsupport/build.gradle | 2 +- .../togglz/FeatureManagerSupport.java | 3 +- .../togglz/FeatureManagerSupportTest.java | 43 +++++++++++++++++++ 5 files changed, 51 insertions(+), 4 deletions(-) create mode 100644 edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc34e311..1ff98102e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Release Notes +## 2.1.0 +* **[edison-togglz]**: + * Make FeatureManager @ConditionalOnMissingBean to allow overriding + * Get @Label and other annotations from FeatureManager.getMetaData so that this works with Features that are not enums, too + * Add methods to FeatureManagerSupport that get the FeatureManager as parameter instead of from the FeatureContext ## 2.0.1 * **[edison-validation]**: Make error profile configurable via application property 'edison.validation.error-profile' diff --git a/build.gradle b/build.gradle index 547263ec8..c7af45bcc 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.0.2-SNAPSHOT' + version = '2.1.0' group = 'de.otto.edison' repositories { diff --git a/edison-testsupport/build.gradle b/edison-testsupport/build.gradle index 3230096f2..15c3ef88e 100644 --- a/edison-testsupport/build.gradle +++ b/edison-testsupport/build.gradle @@ -1,5 +1,5 @@ dependencies { - compileOnly libraries.togglz_console + compile libraries.togglz_console compileOnly libraries.togglz_spring_web compileOnly libraries.togglz_spring_boot_starter compileOnly libraries.aws_sdk_ssm diff --git a/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java b/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java index 99d17da29..7861a36ac 100644 --- a/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java +++ b/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java @@ -4,7 +4,6 @@ import org.togglz.core.context.FeatureContext; import org.togglz.core.manager.FeatureManager; import org.togglz.core.repository.FeatureState; -import org.togglz.core.util.FeatureAnnotations; import static org.togglz.core.context.FeatureContext.clearCache; import static org.togglz.core.context.FeatureContext.getFeatureManager; @@ -52,7 +51,7 @@ public static boolean shouldRunInTests(Feature feature) { } private static boolean shouldRunInTests(FeatureManager featureManager, Feature feature) { - return !featureManager.getMetaData(feature).getAttributes().containsKey("inactiveInTests"); + return !featureManager.getMetaData(feature).getLabel().contains("[inactiveInTests]"); } public static void enable(final Feature feature) { diff --git a/edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java b/edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java new file mode 100644 index 000000000..d6708e911 --- /dev/null +++ b/edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java @@ -0,0 +1,43 @@ +package de.otto.edison.testsupport.togglz; + +import org.junit.jupiter.api.Test; +import org.togglz.core.Feature; +import org.togglz.core.annotation.Label; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.togglz.core.context.FeatureContext.getFeatureManager; + +class FeatureManagerSupportTest { + + @Test + void shouldEnableAllFeaturesThatAreNotInactiveInTests() { + //given + + //when + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + + //then + assertThat(getFeatureManager().isActive(TestFeatures.INACTIVE), is(false)); + assertThat(getFeatureManager().isActive(TestFeatures.ACTIVE), is(true)); + } + + @Test + void shouldDisableAllFeatures() { + //given + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + + //when + FeatureManagerSupport.allDisabledFeatureConfig(TestFeatures.class); + + //then + assertThat(getFeatureManager().isActive(TestFeatures.INACTIVE), is(false)); + assertThat(getFeatureManager().isActive(TestFeatures.ACTIVE), is(false)); + } + + public enum TestFeatures implements Feature { + @Label("should be inactive [inactiveInTests]") + INACTIVE, + ACTIVE, + } +} \ No newline at end of file From ac9761b7b22606eb7928535a4478cf8a92a8ed40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 10 Sep 2019 14:24:03 +0200 Subject: [PATCH 18/78] bump next snapshot Co-authored-by: Florian Torkler --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index c7af45bcc..afca1176c 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.0' + version = '2.1.1-SNAPSHOT' group = 'de.otto.edison' repositories { From 4f740cdf9bace9f4515a3e91a16b240fd66a797e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 10 Sep 2019 14:26:16 +0200 Subject: [PATCH 19/78] use feature names as cache keys in s3TogglzRepo in order to be able to use kotlin feature enums as well Co-authored-by: Florian Torkler --- CHANGELOG.md | 5 +++ .../edison/togglz/s3/S3TogglzRepository.java | 20 +++++++++--- .../togglz/util/FeatureManagerSupport.java | 14 ++++++++ .../util/FeatureManagerSupportTest.java | 32 +++++++++++++++++++ 4 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java create mode 100644 edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ff98102e..dd2559052 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Release Notes +## 2.1.1 +* **[edison-togglz]**: + * Make S3TogglzRepository cache dependent from String instead of Feature in order to support kotlin togglz + * Add `getFeatureFromName(String name)`- function to be able to retrieve current feature by its name + ## 2.1.0 * **[edison-togglz]**: * Make FeatureManager @ConditionalOnMissingBean to allow overriding diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java b/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java index d559a6f0a..a873ab74c 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java @@ -9,8 +9,11 @@ import org.togglz.core.repository.StateRepository; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import static de.otto.edison.togglz.util.FeatureManagerSupport.getFeatureFromName; + /** * Togglz state repository, that fetches the s3 state async * to avoid s3 access while asking for togglz state. @@ -20,7 +23,7 @@ public class S3TogglzRepository implements StateRepository { private final static Logger LOG = LoggerFactory.getLogger(S3TogglzRepository.class); private static final int SCHEDULE_RATE_IN_MILLISECONDS = 60000; - private final Map cache = new ConcurrentHashMap<>(); + private final Map cache = new ConcurrentHashMap<>(); private final FeatureStateConverter featureStateConverter; public S3TogglzRepository(final FeatureStateConverter featureStateConverter) { @@ -30,7 +33,7 @@ public S3TogglzRepository(final FeatureStateConverter featureStateConverter) { @Override public FeatureState getFeatureState(final Feature feature) { - final CacheEntry cachedEntry = cache.get(feature); + final CacheEntry cachedEntry = cache.get(feature.name()); if (cachedEntry != null) { return cachedEntry.getState(); @@ -38,7 +41,7 @@ public FeatureState getFeatureState(final Feature feature) { // no cache hit - refresh state from delegate final FeatureState featureState = featureStateConverter.retrieveFeatureStateFromS3(feature); - cache.put(feature, new CacheEntry(featureState)); + cache.put(feature.name(), new CacheEntry(featureState)); return featureState; } @@ -46,7 +49,7 @@ public FeatureState getFeatureState(final Feature feature) { @Override public void setFeatureState(final FeatureState featureState) { featureStateConverter.persistFeatureStateToS3(featureState); - cache.put(featureState.getFeature(), new CacheEntry(featureState)); + cache.put(featureState.getFeature().name(), new CacheEntry(featureState)); } @Scheduled(initialDelay = 0, fixedRate = SCHEDULE_RATE_IN_MILLISECONDS) @@ -56,7 +59,14 @@ protected void prefetchFeatureStates() { initializeFeatureStates(); } else { LOG.debug("Refreshing state for features"); - cache.replaceAll((feature, cacheEntry) -> new CacheEntry(featureStateConverter.retrieveFeatureStateFromS3(feature))); + + cache.replaceAll((featureName, cacheEntry) -> { + Optional featureFromName = getFeatureFromName(featureName); + if (featureFromName.isPresent()) { + return new CacheEntry(featureStateConverter.retrieveFeatureStateFromS3(featureFromName.get())); + } + return null; + }); } } diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java b/edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java new file mode 100644 index 000000000..6f0f2cd6e --- /dev/null +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java @@ -0,0 +1,14 @@ +package de.otto.edison.togglz.util; + +import org.togglz.core.Feature; + +import java.util.Optional; + +import static org.togglz.core.context.FeatureContext.getFeatureManager; + +public class FeatureManagerSupport { + + public static Optional getFeatureFromName(String name) { + return getFeatureManager().getFeatures().stream().filter(feature -> feature.name().equals(name)).findAny(); + } +} diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java new file mode 100644 index 000000000..178036275 --- /dev/null +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java @@ -0,0 +1,32 @@ +package de.otto.edison.togglz.util; + +import de.otto.edison.togglz.TestFeatures; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.togglz.core.Feature; + +import java.util.Optional; + +import static de.otto.edison.testsupport.togglz.FeatureManagerSupport.allEnabledFeatureConfig; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; + +class FeatureManagerSupportTest { + + @BeforeEach + void setUp() { + allEnabledFeatureConfig(TestFeatures.class); + } + + @Test + void shouldReturnTheCorrectFeature() { + assertThat(FeatureManagerSupport.getFeatureFromName("TEST_FEATURE").get(), is(TestFeatures.TEST_FEATURE)); + } + + @Test + void shouldReturnTheAnEmptyFeatureIfNameisNotKnown() { + Optional unknwonFeature = FeatureManagerSupport.getFeatureFromName("UNKNWON_FEATURE"); + assertFalse(unknwonFeature.isPresent()); + } +} \ No newline at end of file From 6d37fc5c1f64ce537c04b7ae86bc01a837e7af4a Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Mon, 16 Sep 2019 14:13:34 +0200 Subject: [PATCH 20/78] Release 2.1.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index afca1176c..8c9abc721 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.1-SNAPSHOT' + version = '2.1.1' group = 'de.otto.edison' repositories { From 8633c8331c501268251dc00f822b493035f7ab1e Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Mon, 16 Sep 2019 14:42:39 +0200 Subject: [PATCH 21/78] Next snapshot Co-authored-by: Marco Geweke --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8c9abc721..d1ea22867 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.1' + version = '2.1.2-SNAPSHOT' group = 'de.otto.edison' repositories { From 3cb607acab3139854c0c51660aa7f0259c2665fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Mon, 23 Sep 2019 08:06:43 +0200 Subject: [PATCH 22/78] use latest gradle version --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 115e6ac0a..7c4388a92 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 2d0776a3f67295a067861f26ffe5108af36f63ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Mon, 23 Sep 2019 08:39:12 +0200 Subject: [PATCH 23/78] release vesrion 2.1.2 --- CHANGELOG.md | 3 +++ build.gradle | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd2559052..daa46cc04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Release Notes +## 2.1.2 +* **[general]**: increase gradle version + ## 2.1.1 * **[edison-togglz]**: * Make S3TogglzRepository cache dependent from String instead of Feature in order to support kotlin togglz diff --git a/build.gradle b/build.gradle index d1ea22867..5676d6d81 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.2-SNAPSHOT' + version = '2.1.2' group = 'de.otto.edison' repositories { From 1d120f9ba56320100b4e34e71b2881b974534e1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Mon, 23 Sep 2019 08:39:34 +0200 Subject: [PATCH 24/78] bump next snapshot --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5676d6d81..9451606cd 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.2' + version = '2.1.3-SNAPSHOT' group = 'de.otto.edison' repositories { From 6e1c191d11a471cb8cc2f32f8b60554a50ac8ed4 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 25 Sep 2019 09:39:23 +0200 Subject: [PATCH 25/78] Update dependencies Co-authored-by: Marco Geweke --- ...idationExceptionHandlerAcceptanceTest.java | 6 ++--- gradle/dependencies.gradle | 26 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java b/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java index 367b429b5..760d802fd 100644 --- a/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java +++ b/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java @@ -61,9 +61,9 @@ public void shouldValidateAndProduceErrorRepresentation() { .assertThat() .statusCode(is(422)).and() .header("Content-type", Matchers.containsString(";charset=utf-8")) - .content("errors.id[0].key", Collections.emptyList(), is("id.invalid")) - .content("errors.id[0].message", Collections.emptyList(), is("Ungueltiger Id-Wert.")) - .content("errors.id[0].rejected", Collections.emptyList(), is("_!NON_SAFE_ID!!?**")); + .body("errors.id[0].key", Collections.emptyList(), is("id.invalid")) + .body("errors.id[0].message", Collections.emptyList(), is("Ungueltiger Id-Wert.")) + .body("errors.id[0].rejected", Collections.emptyList(), is("_!NON_SAFE_ID!!?**")); } public static class TestConfiguration { diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index b19b27a65..4d6a6c7a8 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -4,18 +4,18 @@ */ ext { versions = [ - spring_boot : '2.1.6.RELEASE', - spring : '5.1.8.RELEASE', - spring_security_core : '5.1.5.RELEASE', - spring_security_web : '5.1.5.RELEASE', + spring_boot : '2.1.8.RELEASE', + spring : '5.1.9.RELEASE', + spring_security_core : '5.1.6.RELEASE', + spring_security_web : '5.1.6.RELEASE', spring_security_oauth : '2.3.6.RELEASE', spring_security_jwt : '1.0.10.RELEASE', - async_http_client : '2.9.0', //Use 2.9.0 because next update updates netty to 4.1.36 which is incompatible with AWS SDK as of 2019-06-02 (needs 4.1.33) + async_http_client : '2.10.2', jcip_annotations : '1.0', logback_classic : '1.2.3', javax_servlet_api : '3.1.0', togglz : '2.6.1.Final', - mongodb_driver : '3.8.1', + mongodb_driver : '3.11.0', caffeine : '2.7.0', json_path : '2.4.0', unboundid_ldapsdk_minimal_edition: '3.2.1', @@ -23,23 +23,23 @@ ext { edison_hal : '2.0.2', validator_collection : '2.2.0', slf4j : '1.7.26', - aws_sdk : '2.6.4', + aws_sdk : '2.9.5', java_validation_api : '2.0.1.Final', java_xml : '2.3.0' ] test_versions = [ - junit : '5.2.0', - hamcrest : '1.3', + junit : '5.5.2', + hamcrest : '2.1', mockito_core : '2.28.2', jsonassert : '1.5.0', - rest_assured : '3.1.1', + rest_assured : '4.1.1', embedded_mongo: '2.2.0', - testcontainers: '1.8.3' + testcontainers: '1.12.1' ] plugin_versions = [ - versions : '0.15.0', + versions : '0.25.0', jacoco : '0.8.2', - nexus_staging: '0.11.0' + nexus_staging: '0.21.1' ] libraries = [ From 387c02ffdbc305a74d481a4c7cb246629e854cca Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 25 Sep 2019 09:48:01 +0200 Subject: [PATCH 26/78] Release 2.1.3 --- CHANGELOG.md | 3 +++ build.gradle | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index daa46cc04..e69181db7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Release Notes +## 2.1.3 +* **[general]**: Update dependencies + ## 2.1.2 * **[general]**: increase gradle version diff --git a/build.gradle b/build.gradle index 9451606cd..da4cfc5e0 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.3-SNAPSHOT' + version = '2.1.3' group = 'de.otto.edison' repositories { From 49270a0a2b3dc8c91572424e96dc8c694da93b34 Mon Sep 17 00:00:00 2001 From: fouquetp Date: Tue, 8 Oct 2019 08:54:21 +0200 Subject: [PATCH 27/78] [edison-jobs]** add retries to manuallyTriggerableJobDefinition --- .../jobs/definition/DefaultJobDefinition.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java b/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java index 6656f2b10..49fa3e0aa 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java @@ -43,6 +43,27 @@ public static JobDefinition manuallyTriggerableJobDefinition(final String jobTyp return new DefaultJobDefinition(jobType, jobName, description, maxAge, Optional.empty(), Optional.empty(), restarts, 0, Optional.empty()); } + /** + * Create a JobDefinition for a job that will not be triggered automatically by a job trigger. + * + * @param jobType The type of the Job + * @param jobName A human readable name of the Job + * @param description A short description of the job's purpose + * @param restarts The number of restarts if the job failed because of errors or exceptions + * @param retries Specifies how often a job trigger should retry to start the job if triggering fails for some reason. + * @param maxAge Optional maximum age of a job. When the job is not run for longer than this duration, + * a warning is displayed on the status page + * @return JobDefinition + */ + public static JobDefinition manuallyTriggerableJobDefinition(final String jobType, + final String jobName, + final String description, + final int restarts, + final int retries, + final Optional maxAge) { + return new DefaultJobDefinition(jobType, jobName, description, maxAge, Optional.empty(), Optional.empty(), restarts, retries, Optional.empty()); + } + /** * Create a JobDefinition that is using a cron expression to specify, when and how often the job should be triggered. * From c02da2dff5a13e3e8b3dc7b7560c91b727784d95 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 25 Sep 2019 11:08:59 +0200 Subject: [PATCH 28/78] Next snapshot Co-authored-by: Marco Geweke --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index da4cfc5e0..873b8a10a 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.3' + version = '2.1.4-SNAPSHOT' group = 'de.otto.edison' repositories { From 31c5a69b48d82166604c69e28056cb453ee762a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Klaus=20Fleerk=C3=B6tter?= Date: Tue, 15 Oct 2019 08:55:44 +0200 Subject: [PATCH 29/78] Ignore unexpected Authorization scheme in Credentials --- .../otto/edison/authentication/Credentials.java | 17 +++++++++++++++-- .../edison/authentication/CredentialsTest.java | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java index 1e7401a2e..fc0dafda1 100644 --- a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java +++ b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java @@ -39,11 +39,24 @@ public static Optional readFrom(HttpServletRequest request) { String authorizationHeader = request.getHeader("Authorization"); if (!StringUtils.isEmpty(authorizationHeader)) { String credentials = authorizationHeader.substring(6, authorizationHeader.length()); - String[] decodedCredentialParts = new String(Base64Utils.decode(credentials.getBytes())).split(":", 2); - if (!decodedCredentialParts[0].isEmpty() && !decodedCredentialParts[1].isEmpty()) { + Optional decodedCredentials = base64Decode(credentials); + String[] decodedCredentialParts = decodedCredentials + .map(s1 -> s1.split(":", 2)) + .orElse(new String[0]); + if (decodedCredentialParts.length >= 2 + && !decodedCredentialParts[0].isEmpty() + && !decodedCredentialParts[1].isEmpty()) { return Optional.of(new Credentials(decodedCredentialParts[0], decodedCredentialParts[1])); } } return Optional.empty(); } + + private static Optional base64Decode(String input) { + try { + return Optional.of(new String(Base64Utils.decode(input.getBytes()))); + } catch (IllegalArgumentException e) { + return Optional.empty(); + } + } } diff --git a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java index 58e755f5c..f834f412f 100644 --- a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java +++ b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java @@ -75,6 +75,20 @@ public void shouldReturnEmptyCredentialsIfUsernameNotSet() { assertThat(credentials.isPresent(), is(false)); } + @Test + public void shouldReturnEmptyCredentialsIfAnotherAuthorizationSchemeThanBasicIsUsed() { + // given + when(httpServletRequest.getHeader("Authorization")) + .thenReturn( + "Token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } + @Test public void shouldReturnCorrectCredentialsIfPasswordContainsColons() { // given From 72758ff41362948291f83a18f92e7fee9a2d1868 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 23 Oct 2019 09:14:49 +0200 Subject: [PATCH 30/78] Update to Spring Boot 2.2 Co-authored-by: Marco Geweke --- build.gradle | 3 +- .../logging/ui/LoggersHtmlEndpoint.java | 2 +- .../StatusControllerAcceptanceTest.java | 2 +- .../ui/DisableEndpointPostProcessorTest.java | 4 +-- .../src/test/resources/application.yml | 10 ++++++- .../main/resources/static/internal/js/jobs.js | 4 +-- .../resources/static/internal/js/logLoader.js | 4 +-- .../src/test/resources/application.yml | 12 ++++++-- edison-oauth/README.md | 2 +- edison-testsupport/build.gradle | 2 ++ .../src/test/resources/application.yml | 9 ++++++ .../web/ValidationExceptionHandler.java | 2 +- ...idationExceptionHandlerAcceptanceTest.java | 2 +- .../src/main/resources/application.yml | 6 +++- .../edison/example/ExampleJobsSmokeTest.java | 8 +++--- .../de/otto/edison/example/ApiController.java | 3 +- .../example/ApiControllerIntegrationTest.java | 17 +++++------ .../src/main/resources/application.yml | 6 +++- .../example/ExampleStatusSmokeTest.java | 4 +-- .../src/main/resources/application.yml | 6 +++- .../example/ExampleTogglzSmokeTest.java | 4 +-- .../src/test/resources/application.yml | 11 +++++++- gradle/dependencies.gradle | 28 +++++++++++-------- gs | 0 24 files changed, 103 insertions(+), 48 deletions(-) delete mode 100644 gs diff --git a/build.gradle b/build.gradle index 873b8a10a..0d1d7f191 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.1.4-SNAPSHOT' + version = '2.2.0-SNAPSHOT' group = 'de.otto.edison' repositories { @@ -50,6 +50,7 @@ subprojects { // Override some Spring Boot default versions // see https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#howto-customize-dependency-versions ext['mockito.version'] = test_versions.mockito_core + ext['jackson.version'] = versions.jackson task allDeps(type: DependencyReportTask) {} diff --git a/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java b/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java index 4108e4498..026519ea0 100644 --- a/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java +++ b/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java @@ -132,7 +132,7 @@ private List> getLoggers() { .keySet() .stream() .map(key -> key.contains("$") ? null : new HashMap() {{ - final LoggerLevels logger = (LoggerLevels) loggers.get(key); + final LoggersEndpoint.SingleLoggerLevels logger = (LoggersEndpoint.SingleLoggerLevels) loggers.get(key); put("name", key); put("displayName", displayNameOf(key)); put("configuredLevel", logger.getConfiguredLevel()); diff --git a/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java b/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java index 2e6538ff3..2d6eb5af2 100644 --- a/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java +++ b/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java @@ -44,7 +44,7 @@ public void shouldGetInternalStatusAsMonitoringStatusJson() throws IOException { then( assertThat(the_status_code().value(), is(200)), - assertThat(the_response_headers().get("Content-Type"), contains("application/vnd.otto.monitoring.status+json;charset=UTF-8")) + assertThat(the_response_headers().get("Content-Type"), contains("application/vnd.otto.monitoring.status+json")) ); } diff --git a/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java b/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java index 66fd5ab43..89046823d 100644 --- a/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java +++ b/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java @@ -46,7 +46,7 @@ public void shouldDisableEndpoint() { @Configuration static class TestEndpointConfiguration { @Bean - Object someTestMvcEndpoint() { + static Object someTestMvcEndpoint() { return new Object(); } } @@ -54,7 +54,7 @@ Object someTestMvcEndpoint() { @Configuration static class RemoveTestEndpointConfiguration { @Bean - DisableEndpointPostProcessor withoutSomeBean() { + static DisableEndpointPostProcessor withoutSomeBean() { return new DisableEndpointPostProcessor("someTest"); } } diff --git a/edison-core/src/test/resources/application.yml b/edison-core/src/test/resources/application.yml index cc16c54aa..65fa40d8d 100644 --- a/edison-core/src/test/resources/application.yml +++ b/edison-core/src/test/resources/application.yml @@ -7,6 +7,8 @@ spring: favor-parameter: true media-types: html: text/html + #jmx: + # enabled: true # context + port of the application server: @@ -14,13 +16,19 @@ server: context-path: /testcore port: 8084 + + # context of the management endpoints like metrics, health, and so on # default is /actuator management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true # edison-specific configuration edison: diff --git a/edison-jobs/src/main/resources/static/internal/js/jobs.js b/edison-jobs/src/main/resources/static/internal/js/jobs.js index 98943108f..4d1331e4a 100644 --- a/edison-jobs/src/main/resources/static/internal/js/jobs.js +++ b/edison-jobs/src/main/resources/static/internal/js/jobs.js @@ -11,8 +11,8 @@ function update() { type: "GET", url: jobsUrl + "?humanReadable=true" + (typeFilter === '' ? '' : "&type=" + typeFilter), headers: { - Accept: "application/json; charset=utf-8", - "Content-Type": "application/json; charset=utf-8" + Accept: "application/json", + "Content-Type": "application/json" }, data: {}, dataType: "json", diff --git a/edison-jobs/src/main/resources/static/internal/js/logLoader.js b/edison-jobs/src/main/resources/static/internal/js/logLoader.js index 7cb94a847..33995a799 100644 --- a/edison-jobs/src/main/resources/static/internal/js/logLoader.js +++ b/edison-jobs/src/main/resources/static/internal/js/logLoader.js @@ -3,8 +3,8 @@ function getLog(logIndex) { type: "GET", url: $('.logWindow').data("job-url"), headers: { - Accept: "application/json; charset=utf-8", - "Content-Type": "application/json; charset=utf-8" + Accept: "application/json", + "Content-Type": "application/json" }, data: {}, dataType: "json", diff --git a/edison-jobs/src/test/resources/application.yml b/edison-jobs/src/test/resources/application.yml index 7feb22365..c48af24f1 100644 --- a/edison-jobs/src/test/resources/application.yml +++ b/edison-jobs/src/test/resources/application.yml @@ -3,8 +3,16 @@ spring: name: testjobs server: - context-path: /testjobs + servlet: + context-path: /testjobs port: 8086 management: - context-path: /internal + endpoints: + web: + base-path: /internal + exposure: + include: '*' + endpoint: + loggers: + enabled: true diff --git a/edison-oauth/README.md b/edison-oauth/README.md index 6b65a6bd0..cfdc9b6eb 100644 --- a/edison-oauth/README.md +++ b/edison-oauth/README.md @@ -67,7 +67,7 @@ Afterwards, you can use an annotation at the controller method that checks the r for a certain `scope` inside the JWT Data: ```java - @RequestMapping(method = GET, value = "/secured/path", produces = MediaType.APPLICATION_JSON_VALUE) + @RequestMapping(method = GET, value = "/secured/path", produces = MediaType.APPLICATION_JSON) @ResponseBody @PreAuthorize("#oauth2.hasScope('some.oauth.scope')") public List getObjects() { diff --git a/edison-testsupport/build.gradle b/edison-testsupport/build.gradle index 15c3ef88e..83a474080 100644 --- a/edison-testsupport/build.gradle +++ b/edison-testsupport/build.gradle @@ -4,6 +4,8 @@ dependencies { compileOnly libraries.togglz_spring_boot_starter compileOnly libraries.aws_sdk_ssm compileOnly libraries.aws_sdk_s3 + compileOnly libraries.jackson_databind + compileOnly libraries.jackson_annotations compileOnly libraries.mongodb_driver compileOnly test_libraries.togglz_testing compileOnly test_libraries.embedded_mongo diff --git a/edison-togglz/src/test/resources/application.yml b/edison-togglz/src/test/resources/application.yml index dd200d49b..732754d85 100644 --- a/edison-togglz/src/test/resources/application.yml +++ b/edison-togglz/src/test/resources/application.yml @@ -8,3 +8,12 @@ server: servlet: context-path: /togglztest port: 8085 + +management: + endpoints: + web: + exposure: + include: '*' + endpoint: + loggers: + enabled: true diff --git a/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java b/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java index dd2389570..d5fdc5c7b 100644 --- a/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java +++ b/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java @@ -15,7 +15,7 @@ public class ValidationExceptionHandler { private static final MediaType APPLICATION_HAL_JSON_ERROR = MediaType.parseMediaType("application/hal+json; " + - "profiles=\"http://spec.otto.de/profiles/error\"; charset=utf-8"); + "profiles=\"http://spec.otto.de/profiles/error\""); private final ErrorHalRepresentationFactory errorHalRepresentationFactory; @Autowired diff --git a/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java b/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java index 760d802fd..b0ea273eb 100644 --- a/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java +++ b/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java @@ -60,7 +60,7 @@ public void shouldValidateAndProduceErrorRepresentation() { .then() .assertThat() .statusCode(is(422)).and() - .header("Content-type", Matchers.containsString(";charset=utf-8")) + .header("Content-type", Matchers.containsString("application/hal+json")) .body("errors.id[0].key", Collections.emptyList(), is("id.invalid")) .body("errors.id[0].message", Collections.emptyList(), is("Ungueltiger Id-Wert.")) .body("errors.id[0].rejected", Collections.emptyList(), is("_!NON_SAFE_ID!!?**")); diff --git a/examples/example-jobs/src/main/resources/application.yml b/examples/example-jobs/src/main/resources/application.yml index 39bf51613..aa8f57902 100644 --- a/examples/example-jobs/src/main/resources/application.yml +++ b/examples/example-jobs/src/main/resources/application.yml @@ -22,7 +22,11 @@ management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true edison: # disable graceful shutdown diff --git a/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java b/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java index e3abe710c..c17ab8021 100644 --- a/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java +++ b/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java @@ -43,21 +43,21 @@ public void shouldRenderMainPage() { public void shouldHaveStatusEndpoint() { final ResponseEntity response = this.restTemplate.getForEntity("/internal/status?format=json", String.class); assertThat(response.getStatusCodeValue()).isEqualTo(200); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getBody()).startsWith("{"); } @Test public void shouldHaveHealthCheck() { final ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isEqualTo(200); } @Test public void shouldHaveJobDefinitions() throws JSONException { final ResponseEntity response = this.restTemplate.getForEntity("/internal/jobdefinitions?format=json", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isEqualTo(200); JSONAssert.assertEquals("{\n" + " \"links\" : [ {\n" + @@ -91,7 +91,7 @@ public void shouldHaveJobDefinitions() throws JSONException { @Test public void shouldHaveFooJobDefinition() throws JSONException { final ResponseEntity response = this.restTemplate.getForEntity("/internal/jobdefinitions/foo?format=json", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isEqualTo(200); JSONAssert.assertEquals("{\n" + " \"type\" : \"Foo\",\n" + diff --git a/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java b/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java index 6c7dc9ad7..f3ef9e9d2 100644 --- a/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java +++ b/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java @@ -1,5 +1,6 @@ package de.otto.edison.example; +import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; @@ -12,7 +13,7 @@ public class ApiController { @RequestMapping( value = "/api/hello", - produces = "application/json", + produces = MediaType.APPLICATION_JSON_VALUE, method = GET) @ResponseBody @PreAuthorize("#oauth2.hasScope('hello.read')") diff --git a/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java b/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java index ae088e024..77a23d261 100644 --- a/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java +++ b/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java @@ -12,6 +12,7 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT; @@ -49,13 +50,13 @@ public void shouldReturnHelloResponseWithValidOauthToken() throws Exception { .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") .addHeader(AUTHORIZATION, bearerToken) - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(200)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), containsString(MediaType.APPLICATION_JSON_VALUE)); assertThat(response.getResponseBody(), is("{\"hello\": \"world\"}")); } @@ -69,13 +70,13 @@ public void shouldReturn403WhenRequestingWithInvalidScopeInOauthToken() throws E .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") .addHeader(AUTHORIZATION, bearerToken) - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(403)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_VALUE)); } @Test @@ -88,13 +89,13 @@ public void shouldReturn403WhenRequestingWithInvalidOauthToken() throws Exceptio .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") .addHeader(AUTHORIZATION, bearerToken) - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(403)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_VALUE)); } @Test @@ -105,13 +106,13 @@ public void shouldReturn403WhenRequestingWithoutOauthToken() throws Exception { final Response response = asyncHttpClient .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(403)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_VALUE)); } diff --git a/examples/example-status/src/main/resources/application.yml b/examples/example-status/src/main/resources/application.yml index da96fea24..cfe67fa89 100644 --- a/examples/example-status/src/main/resources/application.yml +++ b/examples/example-status/src/main/resources/application.yml @@ -15,7 +15,11 @@ management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true edison: gracefulshutdown: diff --git a/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java b/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java index 3a8b83d34..ec39efabe 100644 --- a/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java +++ b/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java @@ -32,14 +32,14 @@ public void shouldRenderMainPage() { public void shouldHaveStatusEndpoint() { final ResponseEntity response = this.restTemplate.getForEntity("/internal/status", String.class); assertThat(response.getStatusCodeValue()).isEqualTo(200); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getBody()).startsWith("{"); } @Test public void shouldHaveHealthCheck() { final ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isIn(200, 503); } diff --git a/examples/example-togglz/src/main/resources/application.yml b/examples/example-togglz/src/main/resources/application.yml index 7b863d211..b585f92ab 100644 --- a/examples/example-togglz/src/main/resources/application.yml +++ b/examples/example-togglz/src/main/resources/application.yml @@ -17,7 +17,11 @@ management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true edison: gracefulshutdown: diff --git a/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java b/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java index 219e30b21..2d9ec53ab 100644 --- a/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java +++ b/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java @@ -39,14 +39,14 @@ public void shouldRenderTogglzConsole() { public void shouldHaveStatusEndpoint() { final ResponseEntity response = this.restTemplate.getForEntity("/internal/status", String.class); assertThat(response.getStatusCodeValue()).isEqualTo(200); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getBody()).startsWith("{"); } @Test public void shouldHaveHealthCheck() { final ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isIn(200, 503); } diff --git a/examples/example-togglz/src/test/resources/application.yml b/examples/example-togglz/src/test/resources/application.yml index e03ff18f1..2753d2cf4 100644 --- a/examples/example-togglz/src/test/resources/application.yml +++ b/examples/example-togglz/src/test/resources/application.yml @@ -1,3 +1,12 @@ spring: main: - allow-bean-definition-overriding: true \ No newline at end of file + allow-bean-definition-overriding: true + +management: + endpoints: + web: + exposure: + include: '*' + endpoint: + loggers: + enabled: true \ No newline at end of file diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 4d6a6c7a8..d9010b471 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -4,35 +4,36 @@ */ ext { versions = [ - spring_boot : '2.1.8.RELEASE', - spring : '5.1.9.RELEASE', - spring_security_core : '5.1.6.RELEASE', - spring_security_web : '5.1.6.RELEASE', - spring_security_oauth : '2.3.6.RELEASE', - spring_security_jwt : '1.0.10.RELEASE', - async_http_client : '2.10.2', + spring_boot : '2.2.0.RELEASE', + spring : '5.2.0.RELEASE', + spring_security_core : '5.2.0.RELEASE', + spring_security_web : '5.2.0.RELEASE', + spring_security_oauth : '2.3.7.RELEASE', + spring_security_jwt : '1.0.11.RELEASE', + async_http_client : '2.10.4', jcip_annotations : '1.0', logback_classic : '1.2.3', javax_servlet_api : '3.1.0', togglz : '2.6.1.Final', mongodb_driver : '3.11.0', - caffeine : '2.7.0', + caffeine : '2.8.0', json_path : '2.4.0', unboundid_ldapsdk_minimal_edition: '3.2.1', hibernate_validator : '6.0.17.Final', edison_hal : '2.0.2', validator_collection : '2.2.0', slf4j : '1.7.26', - aws_sdk : '2.9.5', + aws_sdk : '2.9.24', java_validation_api : '2.0.1.Final', - java_xml : '2.3.0' + java_xml : '2.3.0', + jackson : '2.10.0' ] test_versions = [ junit : '5.5.2', hamcrest : '2.1', mockito_core : '2.28.2', jsonassert : '1.5.0', - rest_assured : '4.1.1', + rest_assured : '4.1.2', embedded_mongo: '2.2.0', testcontainers: '1.12.1' ] @@ -72,7 +73,10 @@ ext { hibernate_validator : "org.hibernate.validator:hibernate-validator:${versions.hibernate_validator}", java_validation_api : "javax.validation:validation-api:${versions.java_validation_api}", edison_hal : "de.otto.edison:edison-hal:${versions.edison_hal}", - validator_collection : "cz.jirutka.validator:validator-collection:${versions.validator_collection}" + validator_collection : "cz.jirutka.validator:validator-collection:${versions.validator_collection}", + jackson : "com.fasterxml.jackson.core:jackson-core:${versions.jackson}", + jackson_databind : "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}", + jackson_annotations : "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" ] java_xml = [ "javax.xml.bind:jaxb-api:${versions.java_xml}", diff --git a/gs b/gs deleted file mode 100644 index e69de29bb..000000000 From 3eded7ce8d3b5dea591701c6021d4fc27a4b6165 Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 23 Oct 2019 09:17:25 +0200 Subject: [PATCH 31/78] Release 2.2.0 Co-authored-by: Marco Geweke --- CHANGELOG.md | 5 +++++ build.gradle | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69181db7..02a6a6711 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Release Notes +## 2.2.0 +* **[general]**: Update to Spring Boot 2.2 + +See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.2-Release-Notes for a migration guide + ## 2.1.3 * **[general]**: Update dependencies diff --git a/build.gradle b/build.gradle index 0d1d7f191..5b84e9ee7 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.0-SNAPSHOT' + version = '2.2.0' group = 'de.otto.edison' repositories { From d6088834145babbcfb7d9174d8a3dc4b46620d7c Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Wed, 23 Oct 2019 09:30:20 +0200 Subject: [PATCH 32/78] Next snapshot Co-authored-by: Marco Geweke --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 5b84e9ee7..8d44b452c 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.0' + version = '2.2.1-SNAPSHOT' group = 'de.otto.edison' repositories { From 3c06e2d6b304d157fe7b62983a5709d50281394f Mon Sep 17 00:00:00 2001 From: Marco Geweke Date: Mon, 28 Oct 2019 15:38:50 +0100 Subject: [PATCH 33/78] New AWS SDK with Jackson 2.10.0 Co-authored-by: Marco Geweke --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index d9010b471..a214a94c1 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -23,7 +23,7 @@ ext { edison_hal : '2.0.2', validator_collection : '2.2.0', slf4j : '1.7.26', - aws_sdk : '2.9.24', + aws_sdk : '2.10.1', java_validation_api : '2.0.1.Final', java_xml : '2.3.0', jackson : '2.10.0' From 4c6a6af2eb9d56d264484e87ea12a3a8416cd4b2 Mon Sep 17 00:00:00 2001 From: cp Date: Wed, 30 Oct 2019 14:37:36 +0100 Subject: [PATCH 34/78] Started working on Dynamo-based JobRepository --- edison-jobs/build.gradle | 2 + .../dynamo/DynamoJobMetaRepository.java | 85 +++++++++++++++++ .../dynamo/DynamoJobRepository.java | 91 +++++++++++++++++++ gradle/dependencies.gradle | 1 + 4 files changed, 179 insertions(+) create mode 100644 edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java create mode 100644 edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java diff --git a/edison-jobs/build.gradle b/edison-jobs/build.gradle index 5b5e55906..7736b743e 100644 --- a/edison-jobs/build.gradle +++ b/edison-jobs/build.gradle @@ -1,6 +1,8 @@ dependencies { compile libraries.async_http_client + compile libraries.aws_sdk_dynamodb + compile project(":edison-core") compileOnly libraries.mongodb_driver diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java new file mode 100644 index 000000000..9c7db234b --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -0,0 +1,85 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobMeta; +import de.otto.edison.jobs.repository.JobMetaRepository; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; +import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.ItemResponse; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class DynamoJobMetaRepository implements JobMetaRepository { + + private DynamoDbClient dynamoDbClient; + + public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { + this.dynamoDbClient = dynamoDbClient; + } + + @Override + public JobMeta getJobMeta(String jobType) { + ImmutableMap key = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName("meta") + .key(key) + .build(); + GetItemResponse response = dynamoDbClient.getItem(itemRequest); + Optional meta = response.getValueForField("meta", JobMeta.class); + return meta.orElse(null); + } + + @Override + public boolean createValue(String jobType, String key, String value) { + return false; + } + + @Override + public boolean setRunningJob(String jobType, String jobId) { + return false; + } + + @Override + public String getRunningJob(String jobType) { + return null; + } + + @Override + public void clearRunningJob(String jobType) { + + } + + @Override + public void disable(String jobType, String comment) { + + } + + @Override + public void enable(String jobType) { + + } + + @Override + public String setValue(String jobType, String key, String value) { + return null; + } + + @Override + public String getValue(String jobType, String key) { + return null; + } + + @Override + public Set findAllJobTypes() { + return null; + } + + @Override + public void deleteAll() { + + } +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java new file mode 100644 index 000000000..cbe94620f --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -0,0 +1,91 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobInfo; +import de.otto.edison.jobs.domain.JobMessage; +import de.otto.edison.jobs.repository.JobRepository; + +import java.time.OffsetDateTime; +import java.util.List; +import java.util.Optional; + +public class DynamoJobRepository implements JobRepository { + @Override + public Optional findOne(String jobId) { + return Optional.empty(); + } + + @Override + public List findLatest(int maxCount) { + return null; + } + + @Override + public List findLatestJobsDistinct() { + return null; + } + + @Override + public List findLatestBy(String type, int maxCount) { + return null; + } + + @Override + public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { + return null; + } + + @Override + public List findAll() { + return null; + } + + @Override + public List findAllJobInfoWithoutMessages() { + return null; + } + + @Override + public List findByType(String jobType) { + return null; + } + + @Override + public JobInfo createOrUpdate(JobInfo job) { + return null; + } + + @Override + public void removeIfStopped(String jobId) { + + } + + @Override + public JobInfo.JobStatus findStatus(String jobId) { + return null; + } + + @Override + public void appendMessage(String jobId, JobMessage jobMessage) { + + } + + @Override + public void setJobStatus(String jobId, JobInfo.JobStatus jobStatus) { + + } + + @Override + public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { + + } + + @Override + public long size() { + return 0; + } + + @Override + public void deleteAll() { + + } +} diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index a214a94c1..f16e80553 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -58,6 +58,7 @@ ext { spring_security_jwt : "org.springframework.security:spring-security-jwt:${versions.spring_security_jwt}", aws_sdk_s3 : "software.amazon.awssdk:s3:${versions.aws_sdk}", aws_sdk_ssm : "software.amazon.awssdk:ssm:${versions.aws_sdk}", + aws_sdk_dynamodb : "software.amazon.awssdk:dynamodb:${versions.aws_sdk}", async_http_client : "org.asynchttpclient:async-http-client:${versions.async_http_client}", jcip_annotations : "net.jcip:jcip-annotations:${versions.jcip_annotations}", logback_classic : "ch.qos.logback:logback-classic:${versions.logback_classic}", From 8035fac8a3815262d0b835e346a83e7bcfa6b007 Mon Sep 17 00:00:00 2001 From: cp Date: Mon, 4 Nov 2019 14:31:55 +0100 Subject: [PATCH 35/78] Finish implementing getMeta for dynamo-based job repository --- .../dynamo/DynamoJobMetaRepository.java | 33 ++++++-- .../dynamo/DynamoJobMetaRepositoryTest.java | 75 +++++++++++++++++++ 2 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index 9c7db234b..fd9d6dcd0 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -1,36 +1,55 @@ package de.otto.edison.jobs.repository.dynamo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.type.MapType; +import com.fasterxml.jackson.databind.type.TypeFactory; import de.otto.edison.jobs.domain.JobMeta; import de.otto.edison.jobs.repository.JobMetaRepository; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; -import software.amazon.awssdk.services.dynamodb.model.ItemResponse; import software.amazon.awssdk.utils.ImmutableMap; +import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.Set; public class DynamoJobMetaRepository implements JobMetaRepository { - private DynamoDbClient dynamoDbClient; + private final MapType mapType; + private final DynamoDbClient dynamoDbClient; - public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { + private final ObjectMapper objectMapper; + + public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient, ObjectMapper objectMapper) { this.dynamoDbClient = dynamoDbClient; + this.objectMapper = objectMapper; + this.mapType = TypeFactory.defaultInstance().constructMapType(HashMap.class, String.class, String.class); } @Override public JobMeta getJobMeta(String jobType) { ImmutableMap key = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); GetItemRequest itemRequest = GetItemRequest.builder() - .tableName("meta") + .tableName("jobMeta") .key(key) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); - Optional meta = response.getValueForField("meta", JobMeta.class); - return meta.orElse(null); + String metaRaw = response.item().get("jobMeta").s(); + + Map meta = null; + try { + meta = objectMapper.readValue(metaRaw, mapType); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + final boolean isRunning = meta.containsKey("running") && Boolean.parseBoolean(meta.get("running")); + final boolean isDisabled = meta.containsKey("disabled") && Boolean.parseBoolean(meta.get("disabled")); + final String comment = meta.get("disabledComment"); + + return new JobMeta(jobType, isRunning, isDisabled, comment, meta); } @Override diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java new file mode 100644 index 000000000..9561cadd4 --- /dev/null +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java @@ -0,0 +1,75 @@ +package de.otto.edison.jobs.repository.dynamo; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import de.otto.edison.jobs.domain.JobMeta; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.AttributeValue; +import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; +import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class DynamoJobMetaRepositoryTest { + + private DynamoJobMetaRepository testee; + + private DynamoDbClient dynamoDbClient; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + objectMapper = new ObjectMapper(); + dynamoDbClient = mock(DynamoDbClient.class); + testee = new DynamoJobMetaRepository(dynamoDbClient, objectMapper); + } + + @ParameterizedTest + @MethodSource("metaInformation") + public void shouldGetJobMetaFromDbWithIncompleteInformation(final ImmutableMap givenMeta, + final boolean isRunning, + final boolean isDisabled, + final String jobType, + final String disableComment) throws JsonProcessingException { + //given + String jsonifiedMeta = objectMapper.writeValueAsString(givenMeta); + GetItemResponse getItemResponse = GetItemResponse.builder() + .item(ImmutableMap.of("jobMeta", AttributeValue.builder().s(jsonifiedMeta).build())) + .build(); + + when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn(getItemResponse); + + //when + JobMeta jobMeta = testee.getJobMeta("someJobType"); + + + //then + assertThat(jobMeta.isRunning(), is(isRunning)); + assertThat(jobMeta.isDisabled(), is(isDisabled)); + assertThat(jobMeta.getJobType(), is(jobType)); + assertThat(jobMeta.getDisabledComment(), is(disableComment)); + assertThat(jobMeta.getAll(), is(givenMeta)); + } + + private static Stream metaInformation() { + return Stream.of( + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "false", "disabled", "false", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "true", "disabled", "true", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabled", "true", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, true, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "false", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "false", "disabled", "false", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "") + ); + } +} \ No newline at end of file From 7b346b031bb8fd4b1ed87ae8504434b6ca2060ba Mon Sep 17 00:00:00 2001 From: cp Date: Mon, 4 Nov 2019 18:09:10 +0100 Subject: [PATCH 36/78] Implement some more methods from job meta repository. --- .../dynamo/DynamoJobMetaRepository.java | 68 +++++++-- .../dynamo/DynamoJobMetaRepositoryTest.java | 138 +++++++++++++++++- 2 files changed, 188 insertions(+), 18 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index fd9d6dcd0..c9336e1c3 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -7,9 +7,7 @@ import de.otto.edison.jobs.domain.JobMeta; import de.otto.edison.jobs.repository.JobMetaRepository; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; -import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.*; import software.amazon.awssdk.utils.ImmutableMap; import java.util.HashMap; @@ -45,8 +43,8 @@ public JobMeta getJobMeta(String jobType) { } catch (JsonProcessingException e) { throw new RuntimeException(e); } - final boolean isRunning = meta.containsKey("running") && Boolean.parseBoolean(meta.get("running")); - final boolean isDisabled = meta.containsKey("disabled") && Boolean.parseBoolean(meta.get("disabled")); + final boolean isRunning = meta.containsKey("running"); + final boolean isDisabled = meta.containsKey("disabled"); final String comment = meta.get("disabledComment"); return new JobMeta(jobType, isRunning, isDisabled, comment, meta); @@ -54,22 +52,38 @@ public JobMeta getJobMeta(String jobType) { @Override public boolean createValue(String jobType, String key, String value) { - return false; + ImmutableMap itemRequestKey = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName("jobMeta") + .key(itemRequestKey) + .build(); + GetItemResponse response = dynamoDbClient.getItem(itemRequest); + + HashMap item = new HashMap<>(response.item()); + item.put(key, AttributeValue.builder().s(value).build()); + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName("jobMeta") + .item(item) + .build(); + + dynamoDbClient.putItem(putItemRequest); + + return true; } @Override public boolean setRunningJob(String jobType, String jobId) { - return false; + return createValue(jobType, "running", jobId); } @Override public String getRunningJob(String jobType) { - return null; + return getValue(jobType, "running"); } @Override public void clearRunningJob(String jobType) { - + setValue(jobType, "running", null); } @Override @@ -84,12 +98,44 @@ public void enable(String jobType) { @Override public String setValue(String jobType, String key, String value) { - return null; + ImmutableMap itemRequestKey = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName("jobMeta") + .key(itemRequestKey) + .build(); + GetItemResponse response = dynamoDbClient.getItem(itemRequest); + String previous = null; + Map responseItem = response.item(); + if (responseItem != null) { + previous = responseItem.get(key) != null ? responseItem.get(key).s() : null; + + HashMap item = new HashMap<>(responseItem); + item.put(key, AttributeValue.builder().s(value).build()); + + if (value == null) { + item.remove(key); + } + + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName("jobMeta") + .item(item) + .build(); + + dynamoDbClient.putItem(putItemRequest); + } + + return previous; } @Override public String getValue(String jobType, String key) { - return null; + ImmutableMap getItemRequestKey = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName("jobMeta") + .key(getItemRequestKey) + .build(); + GetItemResponse response = dynamoDbClient.getItem(itemRequest); + return response.item().get(key).s(); } @Override diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java index 9561cadd4..d2778af57 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import de.otto.edison.jobs.domain.JobMeta; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -11,15 +12,16 @@ import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; +import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; import software.amazon.awssdk.utils.ImmutableMap; +import javax.management.Attribute; import java.util.stream.Stream; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; class DynamoJobMetaRepositoryTest { @@ -62,14 +64,136 @@ public void shouldGetJobMetaFromDbWithIncompleteInformation(final ImmutableMap metaInformation() { return Stream.of( - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "false", "disabled", "false", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "true", "disabled", "true", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabled", "true", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, true, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "false", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId1", "disabled", "disabled", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId2", "disabled", "disabled", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabled", "disabled", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, true, "someJobType", "someDisableComment"), + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId3", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, false, "someJobType", "someDisableComment"), Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "false", "disabled", "false", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "") + Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId4", "disabled", "disabled", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "") ); } } \ No newline at end of file From a0152935463e2823c0d6101265a551ecc0f027e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:31:12 +0100 Subject: [PATCH 37/78] fix credentials retrieval when authentication is not basic or missing a colon --- .../edison/authentication/Credentials.java | 4 +-- .../authentication/CredentialsTest.java | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java index 1e7401a2e..3044e728e 100644 --- a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java +++ b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java @@ -37,10 +37,10 @@ public String getPassword() { */ public static Optional readFrom(HttpServletRequest request) { String authorizationHeader = request.getHeader("Authorization"); - if (!StringUtils.isEmpty(authorizationHeader)) { + if (!StringUtils.isEmpty(authorizationHeader) && authorizationHeader.contains("Basic")) { String credentials = authorizationHeader.substring(6, authorizationHeader.length()); String[] decodedCredentialParts = new String(Base64Utils.decode(credentials.getBytes())).split(":", 2); - if (!decodedCredentialParts[0].isEmpty() && !decodedCredentialParts[1].isEmpty()) { + if (decodedCredentialParts.length == 2 && !decodedCredentialParts[0].isEmpty() && !decodedCredentialParts[1].isEmpty()) { return Optional.of(new Credentials(decodedCredentialParts[0], decodedCredentialParts[1])); } } diff --git a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java index 58e755f5c..a0eb6caa2 100644 --- a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java +++ b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java @@ -88,4 +88,29 @@ public void shouldReturnCorrectCredentialsIfPasswordContainsColons() { assertThat(credentials.get().getUsername(), is("user")); assertThat(credentials.get().getPassword(), is("pass:word")); } + + @Test + public void shouldReturnEmptyCredentialsIfColonDoesNotExist() { + // given + mockHttpServletRequestWithAuthentication("userpass"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } + + @Test + public void shouldReturnEmptyCredentialsIfAuthenticationNotBasic() { + // given + when(httpServletRequest.getHeader("Authorization")) + .thenReturn("Bearer someToken"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } } \ No newline at end of file From 1ffd5ea624f2bd946859ebeab9f08a624375844b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:35:33 +0100 Subject: [PATCH 38/78] update change log for version 2.2.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02a6a6711..8fad9e4ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release Notes +## 2.2.1 +* **[general]**: upgrade aws sdk +* **[edison-core]**: Fix basic auth credentials retrieval on wrong format + ## 2.2.0 * **[general]**: Update to Spring Boot 2.2 From 28c2f2aebabde584735455503e95c882b6169930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:35:59 +0100 Subject: [PATCH 39/78] release version 2.2.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8d44b452c..87021c6ee 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.1-SNAPSHOT' + version = '2.2.1' group = 'de.otto.edison' repositories { From 2ff9dcbb68efc2de15517a55298b0d39aaa06e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:44:29 +0100 Subject: [PATCH 40/78] bump next snapshot --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 87021c6ee..f5b73b204 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.1' + version = '2.2.2-SNAPSHOT' group = 'de.otto.edison' repositories { From 121513e33966fa549203d0cfe7ebbb7e61296a30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:31:12 +0100 Subject: [PATCH 41/78] fix credentials retrieval when authentication is not basic or missing a colon --- .../edison/authentication/Credentials.java | 4 +-- .../authentication/CredentialsTest.java | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java index 1e7401a2e..3044e728e 100644 --- a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java +++ b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java @@ -37,10 +37,10 @@ public String getPassword() { */ public static Optional readFrom(HttpServletRequest request) { String authorizationHeader = request.getHeader("Authorization"); - if (!StringUtils.isEmpty(authorizationHeader)) { + if (!StringUtils.isEmpty(authorizationHeader) && authorizationHeader.contains("Basic")) { String credentials = authorizationHeader.substring(6, authorizationHeader.length()); String[] decodedCredentialParts = new String(Base64Utils.decode(credentials.getBytes())).split(":", 2); - if (!decodedCredentialParts[0].isEmpty() && !decodedCredentialParts[1].isEmpty()) { + if (decodedCredentialParts.length == 2 && !decodedCredentialParts[0].isEmpty() && !decodedCredentialParts[1].isEmpty()) { return Optional.of(new Credentials(decodedCredentialParts[0], decodedCredentialParts[1])); } } diff --git a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java index 58e755f5c..a0eb6caa2 100644 --- a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java +++ b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java @@ -88,4 +88,29 @@ public void shouldReturnCorrectCredentialsIfPasswordContainsColons() { assertThat(credentials.get().getUsername(), is("user")); assertThat(credentials.get().getPassword(), is("pass:word")); } + + @Test + public void shouldReturnEmptyCredentialsIfColonDoesNotExist() { + // given + mockHttpServletRequestWithAuthentication("userpass"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } + + @Test + public void shouldReturnEmptyCredentialsIfAuthenticationNotBasic() { + // given + when(httpServletRequest.getHeader("Authorization")) + .thenReturn("Bearer someToken"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } } \ No newline at end of file From 254e266fd858a2c5cf7d95345d588f9ce5a76fb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:35:33 +0100 Subject: [PATCH 42/78] update change log for version 2.2.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02a6a6711..8fad9e4ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release Notes +## 2.2.1 +* **[general]**: upgrade aws sdk +* **[edison-core]**: Fix basic auth credentials retrieval on wrong format + ## 2.2.0 * **[general]**: Update to Spring Boot 2.2 From 8577d95ad6758b997536404a5c82370c297ea36a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:35:59 +0100 Subject: [PATCH 43/78] release version 2.2.1 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8d44b452c..87021c6ee 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.1-SNAPSHOT' + version = '2.2.1' group = 'de.otto.edison' repositories { From 294d4aee6c3cc00096657de85baf787292471217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 5 Nov 2019 09:44:29 +0100 Subject: [PATCH 44/78] bump next snapshot --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 87021c6ee..f5b73b204 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.1' + version = '2.2.2-SNAPSHOT' group = 'de.otto.edison' repositories { From 677c13d1b89736b3a4f9ed33e91716c1690b1bd2 Mon Sep 17 00:00:00 2001 From: cp Date: Wed, 6 Nov 2019 16:20:40 +0100 Subject: [PATCH 45/78] (rf/os/cp) Implemented JobMeta repository with DynamoDb backend --- edison-jobs/build.gradle | 1 + .../dynamo/DynamoJobMetaRepository.java | 173 ++++++++++----- .../dynamo/DynamoJobMetaRepositoryTest.java | 199 ------------------ .../mongo/JobMetaRepositoryTest.java | 48 ++++- gradle/dependencies.gradle | 5 +- 5 files changed, 165 insertions(+), 261 deletions(-) delete mode 100644 edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java diff --git a/edison-jobs/build.gradle b/edison-jobs/build.gradle index 7736b743e..a9f978e8c 100644 --- a/edison-jobs/build.gradle +++ b/edison-jobs/build.gradle @@ -13,6 +13,7 @@ dependencies { testCompile test_libraries.json_path testCompile test_libraries.jsonassert testCompile test_libraries.embedded_mongo + testImplementation test_libraries.testcontainers } artifacts { diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index c9336e1c3..2863344e9 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -1,9 +1,5 @@ package de.otto.edison.jobs.repository.dynamo; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.type.MapType; -import com.fasterxml.jackson.databind.type.TypeFactory; import de.otto.edison.jobs.domain.JobMeta; import de.otto.edison.jobs.repository.JobMetaRepository; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; @@ -13,138 +9,199 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyMap; public class DynamoJobMetaRepository implements JobMetaRepository { - private final MapType mapType; + private static final String KEY_DISABLED = "_e_disabled"; + private static final String KEY_RUNNING = "_e_running"; private final DynamoDbClient dynamoDbClient; - private final ObjectMapper objectMapper; - - public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient, ObjectMapper objectMapper) { + public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { this.dynamoDbClient = dynamoDbClient; - this.objectMapper = objectMapper; - this.mapType = TypeFactory.defaultInstance().constructMapType(HashMap.class, String.class, String.class); } @Override public JobMeta getJobMeta(String jobType) { - ImmutableMap key = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + ImmutableMap key = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() .tableName("jobMeta") .key(key) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); - String metaRaw = response.item().get("jobMeta").s(); + Map responseItem = response.item(); + if (!responseItem.isEmpty()) { + + final boolean isRunning = responseItem.containsKey(KEY_RUNNING); + final boolean isDisabled = responseItem.containsKey(KEY_DISABLED); + final AttributeValue attributeValueComment = responseItem.get(KEY_DISABLED); + String comment = null; + if (attributeValueComment != null) { + comment = attributeValueComment.s(); + } - Map meta = null; - try { - meta = objectMapper.readValue(metaRaw, mapType); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + Map metaMap = responseItem.entrySet().stream() + .filter(e -> !e.getKey().startsWith("_e_")) + .filter(e -> !e.getKey().equals("jobType")) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().s())); + return new JobMeta(jobType, isRunning, isDisabled, comment, metaMap); + } else { + return new JobMeta(jobType, false, false, "", emptyMap()); } - final boolean isRunning = meta.containsKey("running"); - final boolean isDisabled = meta.containsKey("disabled"); - final String comment = meta.get("disabledComment"); - - return new JobMeta(jobType, isRunning, isDisabled, comment, meta); } @Override public boolean createValue(String jobType, String key, String value) { - ImmutableMap itemRequestKey = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); - GetItemRequest itemRequest = GetItemRequest.builder() - .tableName("jobMeta") - .key(itemRequestKey) - .build(); - GetItemResponse response = dynamoDbClient.getItem(itemRequest); - - HashMap item = new HashMap<>(response.item()); - item.put(key, AttributeValue.builder().s(value).build()); - PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName("jobMeta") - .item(item) - .build(); - - dynamoDbClient.putItem(putItemRequest); - - return true; + if (getValue(jobType, key) == null) { + setValue(jobType, key, value); + return true; + } else { + return false; + } } @Override public boolean setRunningJob(String jobType, String jobId) { - return createValue(jobType, "running", jobId); + return createValue(jobType, KEY_RUNNING, jobId); } @Override public String getRunningJob(String jobType) { - return getValue(jobType, "running"); + return getValue(jobType, KEY_RUNNING); } @Override public void clearRunningJob(String jobType) { - setValue(jobType, "running", null); + setValue(jobType, KEY_RUNNING, null); } @Override public void disable(String jobType, String comment) { + setValue(jobType, KEY_DISABLED, comment != null ? comment : ""); } @Override public void enable(String jobType) { - + setValue(jobType, KEY_DISABLED, null); } @Override public String setValue(String jobType, String key, String value) { - ImmutableMap itemRequestKey = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + putIfAbsent(jobType); + return putValue(jobType, key, value); + } + + private String putValue(String jobType, String key, String value) { + ImmutableMap itemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() .tableName("jobMeta") .key(itemRequestKey) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); + Map newEntry = new HashMap<>(); + + AttributeValue previousAttributeValue = response.item().get(key); String previous = null; - Map responseItem = response.item(); - if (responseItem != null) { - previous = responseItem.get(key) != null ? responseItem.get(key).s() : null; + if (previousAttributeValue != null) { + previous = previousAttributeValue.s(); + } - HashMap item = new HashMap<>(responseItem); - item.put(key, AttributeValue.builder().s(value).build()); + newEntry.putAll(response.item()); + if (value == null) { + newEntry.remove(key); + } else { + newEntry.put(key, toAttributeValue(value)); + } - if (value == null) { - item.remove(key); - } + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName("jobMeta") + .item(newEntry) + .build(); + dynamoDbClient.putItem(putItemRequest); + + return previous; + } + private AttributeValue toAttributeValue(String value) { + return AttributeValue.builder().s(value).build(); + } + + private void putIfAbsent(String jobType) { + ImmutableMap itemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName("jobMeta") + .key(itemRequestKey) + .build(); + GetItemResponse response = dynamoDbClient.getItem(itemRequest); + if (response.item().isEmpty()) { + Map item = new HashMap<>(); + item.put("jobType", toAttributeValue(jobType)); PutItemRequest putItemRequest = PutItemRequest.builder() .tableName("jobMeta") .item(item) .build(); - dynamoDbClient.putItem(putItemRequest); } - - return previous; } @Override public String getValue(String jobType, String key) { - ImmutableMap getItemRequestKey = ImmutableMap.of("jobType", AttributeValue.builder().s(jobType).build()); + ImmutableMap getItemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() .tableName("jobMeta") .key(getItemRequestKey) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); - return response.item().get(key).s(); + AttributeValue value = response.item().get(key); + if (value != null) { + return value.s(); + } else { + return null; + } } @Override public Set findAllJobTypes() { - return null; + ScanRequest scanRequest = ScanRequest.builder() + .tableName("jobMeta") + .attributesToGet("jobType").build(); + ScanResponse scanResponse = dynamoDbClient.scan(scanRequest); + Set jobTypes = scanResponse.items().stream().map(m -> m.get("jobType").s()).collect(Collectors.toSet()); + return jobTypes; } @Override public void deleteAll() { + deleteTable(); + setupSchema(); + } + + public void setupSchema() { + dynamoDbClient.createTable(CreateTableRequest.builder() + .tableName("jobMeta") + .attributeDefinitions(AttributeDefinition.builder() + .attributeName("jobType") + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName("jobType") + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build()); + } + public void deleteTable() { + DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() + .tableName("jobMeta").build(); + dynamoDbClient.deleteTable(deleteTableRequest); } + + } diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java deleted file mode 100644 index d2778af57..000000000 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java +++ /dev/null @@ -1,199 +0,0 @@ -package de.otto.edison.jobs.repository.dynamo; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import de.otto.edison.jobs.domain.JobMeta; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.AttributeValue; -import software.amazon.awssdk.services.dynamodb.model.GetItemRequest; -import software.amazon.awssdk.services.dynamodb.model.GetItemResponse; -import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; -import software.amazon.awssdk.utils.ImmutableMap; - -import javax.management.Attribute; -import java.util.stream.Stream; - -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -class DynamoJobMetaRepositoryTest { - - private DynamoJobMetaRepository testee; - - private DynamoDbClient dynamoDbClient; - private ObjectMapper objectMapper; - - @BeforeEach - void setUp() { - objectMapper = new ObjectMapper(); - dynamoDbClient = mock(DynamoDbClient.class); - testee = new DynamoJobMetaRepository(dynamoDbClient, objectMapper); - } - - @ParameterizedTest - @MethodSource("metaInformation") - public void shouldGetJobMetaFromDbWithIncompleteInformation(final ImmutableMap givenMeta, - final boolean isRunning, - final boolean isDisabled, - final String jobType, - final String disableComment) throws JsonProcessingException { - //given - String jsonifiedMeta = objectMapper.writeValueAsString(givenMeta); - GetItemResponse getItemResponse = GetItemResponse.builder() - .item(ImmutableMap.of("jobMeta", AttributeValue.builder().s(jsonifiedMeta).build())) - .build(); - - when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn(getItemResponse); - - //when - JobMeta jobMeta = testee.getJobMeta("someJobType"); - - - //then - assertThat(jobMeta.isRunning(), is(isRunning)); - assertThat(jobMeta.isDisabled(), is(isDisabled)); - assertThat(jobMeta.getJobType(), is(jobType)); - assertThat(jobMeta.getDisabledComment(), is(disableComment)); - assertThat(jobMeta.getAll(), is(givenMeta)); - } - - @Test - void shouldCreatePutItemRequest() { - //given - when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn(GetItemResponse.builder().build()); - - //when - testee.createValue("someNonExistingJobType", "foo", "bar"); - - //then - verify(dynamoDbClient).putItem(PutItemRequest.builder() - .tableName("jobMeta") - .item(ImmutableMap.of("foo", AttributeValue.builder().s("bar").build())) - .build()); - } - - @Test - void shouldGetValue() { - //given - GetItemRequest getItemRequest = GetItemRequest - .builder() - .tableName("jobMeta") - .key(ImmutableMap.of("jobType", - AttributeValue.builder() - .s("someJobType").build())) - .build(); - when(dynamoDbClient.getItem(eq(getItemRequest))) - .thenReturn(GetItemResponse.builder().item(ImmutableMap.of("foo", AttributeValue.builder().s("bar").build())).build()); - - //when - String value = testee.getValue("someJobType", "foo"); - - //then - assertThat(value, is("bar")); - } - - @Test - void shouldReturnRunningStateOfJob() { - //given - GetItemRequest getItemRequest = GetItemRequest - .builder() - .tableName("jobMeta") - .key(ImmutableMap.of("jobType", - AttributeValue.builder() - .s("someJobType").build())) - .build(); - when(dynamoDbClient.getItem(eq(getItemRequest))) - .thenReturn(GetItemResponse.builder().item(ImmutableMap.of("running", AttributeValue.builder().s("someJobId").build())).build()); - - //when - String isRunning = testee.getRunningJob("someJobType"); - - //then - assertThat(isRunning, is("someJobId")); - - } - - @Test - void shouldSetRunningStateOfJob() { - //given - when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn(GetItemResponse.builder().build()); - - //when - testee.setRunningJob("someJobType", "someJobId"); - - //then - verify(dynamoDbClient).putItem(PutItemRequest.builder() - .tableName("jobMeta") - .item(ImmutableMap.of("running", AttributeValue.builder().s("someJobId").build())) - .build()); - } - - @Test - void shouldClearRunningJob() { - //given - when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn(GetItemResponse.builder().build()); - - //when - testee.clearRunningJob("someJobType"); - - //then - verify(dynamoDbClient).putItem(PutItemRequest.builder() - .tableName("jobMeta") - .build()); - } - - @Test - void shouldUpdateValue() { - //given - when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn( - GetItemResponse.builder() - .item(ImmutableMap.of("foo", AttributeValue.builder().s("qux").build())) - .build()); - - //when - testee.setValue("someJobType", "foo", "bar"); - - //then - verify(dynamoDbClient).putItem(PutItemRequest.builder() - .tableName("jobMeta") - .item(ImmutableMap.of("foo", AttributeValue.builder().s("bar").build())) - .build()); - } - - @Test - void shouldUnsetValue() { - //given - when(dynamoDbClient.getItem(any(GetItemRequest.class))).thenReturn( - GetItemResponse.builder() - .item(ImmutableMap.of("someKey1", AttributeValue.builder().s("someValue1").build(), - "someKey2", AttributeValue.builder().s("someValue2").build())) - .build()); - - //when - testee.setValue("someJobType", "someKey1", null); - - //then - verify(dynamoDbClient).putItem(PutItemRequest.builder() - .tableName("jobMeta") - .item(ImmutableMap.of("someKey2", AttributeValue.builder().s("someValue2").build())) - .build()); - } - - private static Stream metaInformation() { - return Stream.of( - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId1", "disabled", "disabled", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId2", "disabled", "disabled", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabled", "disabled", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, true, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId3", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), true, false, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "disabledComment", "someDisableComment", "lastEntryId", "someLastEntryId"), false, false, "someJobType", "someDisableComment"), - Arguments.of(ImmutableMap.of("jobType", "someJobType", "running", "someJobId4", "disabled", "disabled", "lastEntryId", "someLastEntryId"), true, true, "someJobType", "") - ); - } -} \ No newline at end of file diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index 93a69f5e8..a356796d5 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -2,15 +2,26 @@ import de.otto.edison.jobs.domain.JobMeta; import de.otto.edison.jobs.repository.JobMetaRepository; +import de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository; import de.otto.edison.jobs.repository.inmem.InMemJobMetaRepository; import de.otto.edison.mongo.configuration.MongoProperties; import de.otto.edison.testsupport.mongo.EmbeddedMongoHelper; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import java.io.IOException; +import java.net.URI; import java.util.Collection; import java.util.UUID; @@ -21,27 +32,60 @@ import static org.hamcrest.Matchers.*; import static org.hamcrest.core.Is.is; +@Testcontainers public class JobMetaRepositoryTest { + private static DynamoJobMetaRepository dynamoTestee = null; + @AfterAll public static void teardownMongo() { EmbeddedMongoHelper.stopMongoDB(); } @BeforeAll - public static void startMongo() throws IOException { + public static void initDbs() throws IOException { EmbeddedMongoHelper.startMongoDB(); + dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient()); + } + + @Container + static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + .withExposedPorts(8000); + + String dynamoTableName = "jobMeta"; + + @BeforeEach + public void setUpDynamo() { + dynamoTestee.setupSchema(); + } + + @AfterEach + public void tearDown() { + dynamoTestee.deleteTable(); } public static Collection data() { + DynamoDbClient dynamoBuilder = getDynamoDbClient(); return asList( new MongoJobMetaRepository(EmbeddedMongoHelper.getMongoClient().getDatabase("jobmeta-" + UUID.randomUUID()), "jobmeta", new MongoProperties()), - new InMemJobMetaRepository() + new InMemJobMetaRepository(), + dynamoTestee ); } + private static DynamoDbClient getDynamoDbClient() { + String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + + dynamodb.getMappedPort(8000); + + return DynamoDbClient.builder() + .endpointOverride(URI.create(endpointUri)) + .region(Region.EU_CENTRAL_1) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create("acc", "sec"))).build(); + } + @ParameterizedTest @MethodSource("data") public void shouldStoreAndGetValue(final JobMetaRepository testee) { diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f16e80553..a6c9bbe9a 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -35,7 +35,7 @@ ext { jsonassert : '1.5.0', rest_assured : '4.1.2', embedded_mongo: '2.2.0', - testcontainers: '1.12.1' + testcontainers: '1.12.3' ] plugin_versions = [ versions : '0.25.0', @@ -98,7 +98,8 @@ ext { embedded_mongo : "de.flapdoodle.embed:de.flapdoodle.embed.mongo:${test_versions.embedded_mongo}", json_path : "com.jayway.jsonpath:json-path:${versions.json_path}", rest_assured : "io.rest-assured:rest-assured:${test_versions.rest_assured}", - testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}" +// testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}", + testcontainers : "org.testcontainers:junit-jupiter:${test_versions.testcontainers}" ] gradle_plugins = [ From 5c90821c11da55983d9804ce6dd45519161869a5 Mon Sep 17 00:00:00 2001 From: Oliver Steenbuck Date: Thu, 7 Nov 2019 16:27:05 +0100 Subject: [PATCH 46/78] tm/os|task|implement DynamoJobRepository createOrUpdate/findAll/get --- .../dynamo/DynamoJobRepository.java | 151 ++++++- .../jobs/repository/dynamo/JobStructure.java | 31 ++ .../dynamo/DynamoJobRepositoryTest.java | 384 ++++++++++++++++++ 3 files changed, 563 insertions(+), 3 deletions(-) create mode 100644 edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java create mode 100644 edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index cbe94620f..17320b1f7 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -1,19 +1,52 @@ package de.otto.edison.jobs.repository.dynamo; +import com.fasterxml.jackson.databind.ObjectMapper; import de.otto.edison.jobs.domain.JobInfo; import de.otto.edison.jobs.domain.JobMessage; +import de.otto.edison.jobs.domain.Level; import de.otto.edison.jobs.repository.JobRepository; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; import java.time.OffsetDateTime; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; public class DynamoJobRepository implements JobRepository { + + public static final String JOBS_TABLENAME = "jobs"; + private final DynamoDbClient dynamoDbClient; + private final ObjectMapper objectMapper; + + public DynamoJobRepository(DynamoDbClient dynamoDbClient, ObjectMapper objectMapper) { + this.dynamoDbClient = dynamoDbClient; + this.objectMapper = objectMapper; + } + @Override public Optional findOne(String jobId) { - return Optional.empty(); + Map keyMap = new HashMap<>(); + keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); + + GetItemRequest jobInfoRequest = GetItemRequest.builder() + .tableName(JOBS_TABLENAME) + .key(keyMap) + .build(); + final GetItemResponse jobInfoResponse = dynamoDbClient.getItem(jobInfoRequest); + if (jobInfoResponse.item().isEmpty()) { + return Optional.empty(); + } + final JobInfo jobInfo = decode(jobInfoResponse.item()); + return Optional.of(jobInfo); + } + @Override public List findLatest(int maxCount) { return null; @@ -36,7 +69,11 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { @Override public List findAll() { - return null; + ScanRequest findAll = ScanRequest.builder() + .tableName(JOBS_TABLENAME) + .build(); + final ScanResponse scan = dynamoDbClient.scan(findAll); + return scan.items().stream().map(this::decode).collect(Collectors.toList()); } @Override @@ -51,7 +88,63 @@ public List findByType(String jobType) { @Override public JobInfo createOrUpdate(JobInfo job) { - return null; + + Map jobAsItem = encode(job); + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName(JOBS_TABLENAME) + .item(jobAsItem) + .build(); + dynamoDbClient.putItem(putItemRequest); + + return job; + + } + + private Map encode(JobInfo jobInfo) { + Map jobAsItem = new HashMap<>(); + jobAsItem.put(JobStructure.ID.key(), toStringAttributeValue(jobInfo.getJobId())); + jobAsItem.put(JobStructure.HOSTNAME.key(), toStringAttributeValue(jobInfo.getHostname())); + jobAsItem.put(JobStructure.JOB_TYPE.key(), toStringAttributeValue(jobInfo.getJobType())); + jobAsItem.put(JobStructure.STARTED.key(), toStringAttributeValue(jobInfo.getStarted())); + jobAsItem.put(JobStructure.STATUS.key(), toStringAttributeValue(jobInfo.getStatus().name())); + jobInfo.getStopped().ifPresent(offsetDateTime -> jobAsItem.put(JobStructure.STOPPED.key(), toStringAttributeValue(offsetDateTime))); + jobAsItem.put(JobStructure.LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); + jobAsItem.put(JobStructure.MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); + + + return jobAsItem; + } + + private JobInfo decode(Map item) { + final JobInfo.Builder jobInfo = JobInfo.builder() + .setJobId(item.get(JobStructure.ID.key()).s()) + .setHostname(item.get(JobStructure.HOSTNAME.key()).s()) + .setJobType(item.get(JobStructure.JOB_TYPE.key()).s()) + .setStarted(OffsetDateTime.parse(item.get(JobStructure.STARTED.key()).s())) + .setStatus(JobInfo.JobStatus.valueOf(item.get(JobStructure.STATUS.key()).s())) + .setLastUpdated(OffsetDateTime.parse(item.get(JobStructure.LAST_UPDATED.key()).s())) + .setMessages(itemToJobMessages(item)); + if(item.containsKey(JobStructure.STOPPED.key())){ + jobInfo.setStopped(OffsetDateTime.parse(item.get(JobStructure.STOPPED.key()).s())); + } + return jobInfo.build(); + } + + private List itemToJobMessages(Map item){ + if(!item.containsKey(JobStructure.MESSAGES.key())){ + return emptyList(); + } + + final AttributeValue attributeValue = item.get(JobStructure.MESSAGES.key()); + return attributeValue.l().stream().map(this::attributeValueToMessage).collect(Collectors.toList()); + } + + private JobMessage attributeValueToMessage(AttributeValue attributeValue) { + final Map messageMap = attributeValue.m(); + final Level level = Level.ofKey(messageMap.get(JobStructure.MSG_LEVEL.key()).s()); + final String text = messageMap.get(JobStructure.MSG_TEXT.key()).s(); + final OffsetDateTime timestamp = OffsetDateTime.parse(messageMap.get(JobStructure.MSG_TS.key()).s()); + return JobMessage.jobMessage(level, text, timestamp); } @Override @@ -88,4 +181,56 @@ public long size() { public void deleteAll() { } + + private AttributeValue toStringAttributeValue(OffsetDateTime value) { + return toStringAttributeValue(value.toString()); + } + + private AttributeValue toStringAttributeValue(String value) { + return AttributeValue.builder().s(value).build(); + } + + private AttributeValue toMapAttributeValue(JobMessage jobMessage) { + Map message = new HashMap<>(); + message.put(JobStructure.MSG_LEVEL.key(), toStringAttributeValue(jobMessage.getLevel().getKey())); + message.put(JobStructure.MSG_TEXT.key(), toStringAttributeValue(jobMessage.getMessage())); + message.put(JobStructure.MSG_TS.key(), toStringAttributeValue(jobMessage.getTimestamp())); + return AttributeValue.builder() + .m(message) + .build(); + } + + private AttributeValue messagesToAttributeValueList(List jobeMessages) { + final List messageAttributes = jobeMessages.stream().map(this::toMapAttributeValue).collect(Collectors.toList()); + return toAttributeValueList(messageAttributes); + } + + private AttributeValue toAttributeValueList(List values) { + return AttributeValue.builder().l(values).build(); + } + + public void setupSchema() { + dynamoDbClient.createTable(CreateTableRequest.builder() + .tableName(JOBS_TABLENAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName(JobStructure.ID.key()) + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName(JobStructure.ID.key()) + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build()); + } + + public void deleteTable() { + DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() + .tableName(JOBS_TABLENAME).build(); + dynamoDbClient.deleteTable(deleteTableRequest); + } + } diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java new file mode 100644 index 000000000..a67ebecf6 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java @@ -0,0 +1,31 @@ +package de.otto.edison.jobs.repository.dynamo; + +enum JobStructure { + + ID("jobId"), + STARTED("started"), + STOPPED("stopped"), + JOB_TYPE("type"), + STATUS("status"), + MESSAGES("messages"), + MSG_TS("ts"), + MSG_TEXT("msg"), + MSG_LEVEL("level"), + HOSTNAME("hostname"), + LAST_UPDATED("lastUpdated"); + + private final String key; + + private JobStructure(final String key) { + this.key = key; + } + + public String key() { + return key; + } + + public String toString() { + return key; + } + +} diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java new file mode 100644 index 000000000..dbe57b256 --- /dev/null +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -0,0 +1,384 @@ +package de.otto.edison.jobs.repository.dynamo; + +import com.fasterxml.jackson.databind.ObjectMapper; +import de.otto.edison.jobs.domain.JobInfo; +import de.otto.edison.jobs.domain.JobInfo.JobStatus; +import de.otto.edison.jobs.domain.JobMessage; +import de.otto.edison.jobs.domain.Level; +import de.otto.edison.testsupport.mongo.EmbeddedMongoHelper; +import org.hamcrest.Matchers; +import org.hamcrest.collection.IsCollectionWithSize; +import org.junit.jupiter.api.*; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import java.io.IOException; +import java.net.URI; +import java.time.Clock; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Optional; + +import static de.otto.edison.jobs.domain.JobInfo.JobStatus.ERROR; +import static de.otto.edison.jobs.domain.JobInfo.JobStatus.OK; +import static de.otto.edison.jobs.domain.JobInfo.builder; +import static de.otto.edison.jobs.domain.JobInfo.newJobInfo; +import static de.otto.edison.jobs.domain.JobMessage.jobMessage; +import static de.otto.edison.testsupport.matcher.OptionalMatchers.isAbsent; +import static de.otto.edison.testsupport.matcher.OptionalMatchers.isPresent; +import static java.time.Clock.fixed; +import static java.time.Clock.systemDefaultZone; +import static java.time.OffsetDateTime.now; +import static java.time.ZoneId.systemDefault; +import static java.util.Arrays.asList; +import static java.util.UUID.randomUUID; +import static org.assertj.core.util.Lists.emptyList; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +@Testcontainers +public class DynamoJobRepositoryTest { + + private static DynamoJobRepository testee; + private static ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeAll + public static void initDbs() throws IOException { + testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + } + + @Container + static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + .withExposedPorts(8000); + + + @BeforeEach + public void setUpDynamo() { + testee.setupSchema(); + } + + @AfterEach + public void tearDown() { + testee.deleteTable(); + } + + @BeforeEach + public void setUp() throws Exception { + testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + } + + private static DynamoDbClient getDynamoDbClient() { + String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + + dynamodb.getMappedPort(8000); + + return DynamoDbClient.builder() + .endpointOverride(URI.create(endpointUri)) + .region(Region.EU_CENTRAL_1) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create("acc", "sec"))).build(); + } + + + private Clock clock = systemDefaultZone(); + + @Test + public void shouldCreateOrUpdateJob() { + //Given + final JobInfo savedjobInfo = jobInfo("http://localhost/foo", "T_FOO"); + testee.createOrUpdate(savedjobInfo); + + //When + testee.createOrUpdate(savedjobInfo); + + final Optional readJobInfo = testee.findOne(savedjobInfo.getJobId()); + + //Then + assertThat(readJobInfo.get(), equalTo(savedjobInfo)); + } + +// @Test +// public void shouldFindJobInfoByUri() { +// // given +// DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); +// +// // when +// JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); +// repository.createOrUpdate(job); +// +// // then +// assertThat(repository.findOne(job.getJobId()), isPresent()); +// } +// +// @Test +// public void shouldReturnAbsentStatus() { +// DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); +// assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); +// } +// +// @Test +// public void shouldNotRemoveRunningJobs() { +// // given +// final String testUri = "test"; +// testee.createOrUpdate(newJobInfo(testUri, "FOO", systemDefaultZone(), "localhost")); +// // when +// testee.removeIfStopped(testUri); +// // then +// assertThat(testee.size(), is(1L)); +// } +// +// @Test +// public void shouldNotFailToRemoveMissingJob() { +// // when +// testee.removeIfStopped("foo"); +// // then +// // no Exception is thrown... +// } +// +// @Test +// public void shouldRemoveJob() throws Exception { +// JobInfo stoppedJob = builder() +// .setJobId("some/job/stopped") +// .setJobType("test") +// .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) +// .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) +// .setHostname("localhost") +// .setStatus(JobStatus.OK) +// .build(); +// testee.createOrUpdate(stoppedJob); +// testee.createOrUpdate(stoppedJob); +// +// testee.removeIfStopped(stoppedJob.getJobId()); +// +// assertThat(testee.size(), is(0L)); +// } +// + @Test + public void shouldFindAll() { + // given + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final List jobInfos = testee.findAll(); + // then + assertThat(jobInfos.size(), is(2)); + assertThat(jobInfos.get(0).getJobId(), is("youngest")); + assertThat(jobInfos.get(1).getJobId(), is("oldest")); + } +// +// @Test +// public void shouldFindLatestDistinct() throws Exception { +// // Given +// Instant now = Instant.now(); +// final JobInfo eins = newJobInfo("eins", "eins", fixed(now.plusSeconds(10), systemDefault()), "localhost"); +// final JobInfo zwei = newJobInfo("zwei", "eins", fixed(now.plusSeconds(20), systemDefault()), "localhost"); +// final JobInfo drei = newJobInfo("drei", "zwei", fixed(now.plusSeconds(30), systemDefault()), "localhost"); +// final JobInfo vier = newJobInfo("vier", "drei", fixed(now.plusSeconds(40), systemDefault()), "localhost"); +// final JobInfo fuenf = newJobInfo("fuenf", "drei", fixed(now.plusSeconds(50), systemDefault()), "localhost"); +// +// testee.createOrUpdate(eins); +// testee.createOrUpdate(zwei); +// testee.createOrUpdate(drei); +// testee.createOrUpdate(vier); +// testee.createOrUpdate(fuenf); +// +// // When +// List latestDistinct = testee.findLatestJobsDistinct(); +// +// // Then +// assertThat(latestDistinct, hasSize(3)); +// assertThat(latestDistinct, Matchers.containsInAnyOrder(fuenf, zwei, drei)); +// } +// +// +// +// @Test +// public void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() throws Exception { +// // given +// testee.createOrUpdate(newJobInfo("deadJob", "FOO", fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); +// testee.createOrUpdate(newJobInfo("running", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); +// +// // when +// final List jobInfos = testee.findRunningWithoutUpdateSince(now().minus(5, ChronoUnit.SECONDS)); +// +// // then +// assertThat(jobInfos, IsCollectionWithSize.hasSize(1)); +// assertThat(jobInfos.get(0).getJobId(), is("deadJob")); +// } +// +// @Test +// public void shouldFindLatestByType() { +// // given +// final String type = "TEST"; +// final String otherType = "OTHERTEST"; +// +// +// testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); +// testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); +// testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); +// +// // when +// final List jobInfos = testee.findLatestBy(type, 2); +// +// // then +// assertThat(jobInfos.get(0).getJobId(), is("youngest")); +// assertThat(jobInfos.get(1).getJobId(), is("oldest")); +// assertThat(jobInfos, hasSize(2)); +// } +// +// @Test +// public void shouldFindLatest() { +// // given +// final String type = "TEST"; +// final String otherType = "OTHERTEST"; +// testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); +// testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); +// testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); +// +// // when +// final List jobInfos = testee.findLatest(2); +// +// // then +// assertThat(jobInfos.get(0).getJobId(), is("youngest")); +// assertThat(jobInfos.get(1).getJobId(), is("other")); +// assertThat(jobInfos, hasSize(2)); +// } +// +// @Test +// public void shouldFindAllJobsOfSpecificType() throws Exception { +// // Given +// final String type = "TEST"; +// final String otherType = "OTHERTEST"; +// testee.createOrUpdate(builder() +// .setJobId("1") +// .setJobType(type) +// .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) +// .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) +// .setHostname("localhost") +// .setStatus(JobStatus.OK) +// .build()); +// testee.createOrUpdate(newJobInfo("2", otherType, systemDefaultZone(), "localhost")); +// testee.createOrUpdate(newJobInfo("3", type, systemDefaultZone(), "localhost")); +// +// // When +// final List jobsType1 = testee.findByType(type); +// final List jobsType2 = testee.findByType(otherType); +// +// // Then +// assertThat(jobsType1.size(), is(2)); +// assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("1")), is(true)); +// assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("3")), is(true)); +// assertThat(jobsType2.size(), is(1)); +// assertThat(jobsType2.stream().anyMatch(job -> job.getJobId().equals("2")), is(true)); +// } +// +// @Test +// public void shouldFindStatusOfJob() throws Exception { +// //Given +// final String type = "TEST"; +// JobInfo jobInfo = newJobInfo("1", type, systemDefaultZone(), "localhost"); +// testee.createOrUpdate(jobInfo); +// +// //When +// JobStatus status = testee.findStatus("1"); +// +// //Then +// assertThat(status, is(JobStatus.OK)); +// } +// +// @Test +// public void shouldAppendMessageToJobInfo() throws Exception { +// +// String someUri = "someUri"; +// OffsetDateTime now = now(); +// +// //Given +// JobInfo jobInfo = newJobInfo(someUri, "TEST", systemDefaultZone(), "localhost"); +// testee.createOrUpdate(jobInfo); +// +// //When +// JobMessage igelMessage = JobMessage.jobMessage(Level.WARNING, "Der Igel ist froh.", now); +// testee.appendMessage(someUri, igelMessage); +// +// //Then +// JobInfo jobInfoFromRepo = testee.findOne(someUri).get(); +// +// assertThat(jobInfoFromRepo.getMessages().size(), is(1)); +// assertThat(jobInfoFromRepo.getMessages().get(0), is(igelMessage)); +// assertThat(jobInfoFromRepo.getLastUpdated(), is(now.truncatedTo(ChronoUnit.MILLIS))); +// +// } +// +// @Test +// public void shouldUpdateJobStatus() { +// //Given +// final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); //default jobStatus is 'OK' +// testee.createOrUpdate(foo); +// +// //When +// testee.setJobStatus(foo.getJobId(), ERROR); +// JobStatus status = testee.findStatus("http://localhost/foo"); +// +// //Then +// assertThat(status, is(ERROR)); +// } +// +// +// +// @Test +// public void shouldUpdateJobLastUpdateTime() { +// //Given +// final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); +// testee.createOrUpdate(foo); +// +// OffsetDateTime myTestTime = OffsetDateTime.of(1979, 2, 5, 1, 2, 3, 1_000_000, ZoneOffset.UTC); +// +// //When +// testee.setLastUpdate(foo.getJobId(), myTestTime); +// +// final Optional jobInfo = testee.findOne(foo.getJobId()); +// +// //Then +// assertThat(jobInfo.get().getLastUpdated(), is(myTestTime)); +// } +// +// @Test +// public void shouldClearJobInfos() throws Exception { +// //Given +// JobInfo stoppedJob = builder() +// .setJobId("some/job/stopped") +// .setJobType("test") +// .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) +// .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) +// .setHostname("localhost") +// .setStatus(JobStatus.OK) +// .build(); +// testee.createOrUpdate(stoppedJob); +// +// //When +// testee.deleteAll(); +// +// //Then +// assertThat(testee.findAll(), is(emptyList())); +// } + + private JobInfo jobInfo(final String jobId, final String type) { + return JobInfo.newJobInfo( + jobId, + type, + now(), now(), Optional.of(now()), OK, + asList( + jobMessage(Level.INFO, "foo", now()), + jobMessage(Level.WARNING, "bar", now())), + systemDefaultZone(), + "localhost" + ); + } +} From 85926766f330bd83d06fc6ab92a554e1bc42fb71 Mon Sep 17 00:00:00 2001 From: Oliver Steenbuck Date: Fri, 8 Nov 2019 15:59:03 +0100 Subject: [PATCH 47/78] add size method --- .../dynamo/DynamoJobRepository.java | 39 +++++- .../dynamo/DynamoJobRepositoryTest.java | 121 +++++++++++------- 2 files changed, 108 insertions(+), 52 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 17320b1f7..849d6ba31 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -108,7 +108,9 @@ private Map encode(JobInfo jobInfo) { jobAsItem.put(JobStructure.STARTED.key(), toStringAttributeValue(jobInfo.getStarted())); jobAsItem.put(JobStructure.STATUS.key(), toStringAttributeValue(jobInfo.getStatus().name())); jobInfo.getStopped().ifPresent(offsetDateTime -> jobAsItem.put(JobStructure.STOPPED.key(), toStringAttributeValue(offsetDateTime))); - jobAsItem.put(JobStructure.LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); + if (null != jobInfo.getLastUpdated()) { + jobAsItem.put(JobStructure.LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); + } jobAsItem.put(JobStructure.MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); @@ -122,16 +124,19 @@ private JobInfo decode(Map item) { .setJobType(item.get(JobStructure.JOB_TYPE.key()).s()) .setStarted(OffsetDateTime.parse(item.get(JobStructure.STARTED.key()).s())) .setStatus(JobInfo.JobStatus.valueOf(item.get(JobStructure.STATUS.key()).s())) - .setLastUpdated(OffsetDateTime.parse(item.get(JobStructure.LAST_UPDATED.key()).s())) .setMessages(itemToJobMessages(item)); - if(item.containsKey(JobStructure.STOPPED.key())){ + if (item.containsKey(JobStructure.STOPPED.key())) { jobInfo.setStopped(OffsetDateTime.parse(item.get(JobStructure.STOPPED.key()).s())); } + + if (item.containsKey(JobStructure.LAST_UPDATED.key())) { + jobInfo.setLastUpdated(OffsetDateTime.parse(item.get(JobStructure.LAST_UPDATED.key()).s())); + } return jobInfo.build(); } - private List itemToJobMessages(Map item){ - if(!item.containsKey(JobStructure.MESSAGES.key())){ + private List itemToJobMessages(Map item) { + if (!item.containsKey(JobStructure.MESSAGES.key())) { return emptyList(); } @@ -150,6 +155,13 @@ private JobMessage attributeValueToMessage(AttributeValue attributeValue) { @Override public void removeIfStopped(String jobId) { + Map keyMap = new HashMap<>(); + keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); + DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() + .tableName(JOBS_TABLENAME) + .key(keyMap) + .build(); + dynamoDbClient.deleteItem(deleteJobRequest); } @Override @@ -174,7 +186,22 @@ public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { @Override public long size() { - return 0; + Map lastKeyEvaluated = null; + long count = 0; + do { + ScanRequest counterQuery = ScanRequest.builder() + .tableName(JOBS_TABLENAME) + .select(Select.COUNT) + .limit(2) + .exclusiveStartKey(lastKeyEvaluated) + .build(); + + final ScanResponse countResponse = dynamoDbClient.scan(counterQuery); + lastKeyEvaluated = countResponse.lastEvaluatedKey(); + count = count + countResponse.count(); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + + return count; } @Override diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index dbe57b256..5f6d037c1 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -104,25 +104,33 @@ public void shouldCreateOrUpdateJob() { assertThat(readJobInfo.get(), equalTo(savedjobInfo)); } -// @Test -// public void shouldFindJobInfoByUri() { -// // given -// DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); -// -// // when -// JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); -// repository.createOrUpdate(job); -// -// // then -// assertThat(repository.findOne(job.getJobId()), isPresent()); -// } -// -// @Test -// public void shouldReturnAbsentStatus() { -// DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); -// assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); -// } -// + @Test + public void shouldFindJobInfoByUri() { + // given + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + + // when + JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); + repository.createOrUpdate(job); + + // then + assertThat(repository.findOne(job.getJobId()), isPresent()); + } + + @Test + public void shouldReturnAbsentStatus() { + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); + } + + @Test + public void shouldNotFailToRemoveMissingJob() { + // when + testee.removeIfStopped("foo"); + // then + // no Exception is thrown... + } + // @Test // public void shouldNotRemoveRunningJobs() { // // given @@ -132,34 +140,27 @@ public void shouldCreateOrUpdateJob() { // testee.removeIfStopped(testUri); // // then // assertThat(testee.size(), is(1L)); + // } -// -// @Test -// public void shouldNotFailToRemoveMissingJob() { -// // when -// testee.removeIfStopped("foo"); -// // then -// // no Exception is thrown... -// } -// -// @Test -// public void shouldRemoveJob() throws Exception { -// JobInfo stoppedJob = builder() -// .setJobId("some/job/stopped") -// .setJobType("test") -// .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) -// .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) -// .setHostname("localhost") -// .setStatus(JobStatus.OK) -// .build(); -// testee.createOrUpdate(stoppedJob); -// testee.createOrUpdate(stoppedJob); -// -// testee.removeIfStopped(stoppedJob.getJobId()); -// -// assertThat(testee.size(), is(0L)); -// } -// + + @Test + public void shouldRemoveJob() throws Exception { + JobInfo stoppedJob = builder() + .setJobId("some/job/stopped") + .setJobType("test") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build(); + testee.createOrUpdate(stoppedJob); + testee.createOrUpdate(stoppedJob); + + testee.removeIfStopped(stoppedJob.getJobId()); + + assertThat(testee.size(), is(0L)); + } + @Test public void shouldFindAll() { // given @@ -172,6 +173,34 @@ public void shouldFindAll() { assertThat(jobInfos.get(0).getJobId(), is("youngest")); assertThat(jobInfos.get(1).getJobId(), is("oldest")); } + + @Test + public void shouldFindAllinSizeOperation() { + // given + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final long count = testee.size(); + // then + assertThat(count, is(2L)); + } + + @Test + public void shouldFindAllinSizeOperationWithPageing() { + // given + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youn44444556gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("yo121ungest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youn333333gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youn1212gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final long count = testee.size(); + // then + //TODO write better test with paging enabled in scan + assertThat(count, is(7L)); + } + // // @Test // public void shouldFindLatestDistinct() throws Exception { From 8e77d9acd4386e69c434f7f1602d5bf38ef282c4 Mon Sep 17 00:00:00 2001 From: Oliver Steenbuck Date: Mon, 11 Nov 2019 09:23:41 +0100 Subject: [PATCH 48/78] add size() and tests for page size (findAll() test still failing) --- .../dynamo/DynamoJobRepository.java | 7 ++-- .../dynamo/DynamoJobRepositoryTest.java | 36 +++++++++++-------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 849d6ba31..16319601d 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -22,10 +22,12 @@ public class DynamoJobRepository implements JobRepository { public static final String JOBS_TABLENAME = "jobs"; private final DynamoDbClient dynamoDbClient; private final ObjectMapper objectMapper; + private final int pageSize; - public DynamoJobRepository(DynamoDbClient dynamoDbClient, ObjectMapper objectMapper) { + public DynamoJobRepository(DynamoDbClient dynamoDbClient, ObjectMapper objectMapper, int pageSize) { this.dynamoDbClient = dynamoDbClient; this.objectMapper = objectMapper; + this.pageSize = pageSize; } @Override @@ -71,6 +73,7 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { public List findAll() { ScanRequest findAll = ScanRequest.builder() .tableName(JOBS_TABLENAME) + .limit(pageSize) .build(); final ScanResponse scan = dynamoDbClient.scan(findAll); return scan.items().stream().map(this::decode).collect(Collectors.toList()); @@ -192,7 +195,7 @@ public long size() { ScanRequest counterQuery = ScanRequest.builder() .tableName(JOBS_TABLENAME) .select(Select.COUNT) - .limit(2) + .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .build(); diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 5f6d037c1..5e01c9639 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -3,11 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import de.otto.edison.jobs.domain.JobInfo; import de.otto.edison.jobs.domain.JobInfo.JobStatus; -import de.otto.edison.jobs.domain.JobMessage; import de.otto.edison.jobs.domain.Level; -import de.otto.edison.testsupport.mongo.EmbeddedMongoHelper; -import org.hamcrest.Matchers; -import org.hamcrest.collection.IsCollectionWithSize; import org.junit.jupiter.api.*; import org.testcontainers.containers.GenericContainer; import org.testcontainers.junit.jupiter.Container; @@ -21,13 +17,9 @@ import java.net.URI; import java.time.Clock; import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; -import static de.otto.edison.jobs.domain.JobInfo.JobStatus.ERROR; import static de.otto.edison.jobs.domain.JobInfo.JobStatus.OK; import static de.otto.edison.jobs.domain.JobInfo.builder; import static de.otto.edison.jobs.domain.JobInfo.newJobInfo; @@ -40,7 +32,6 @@ import static java.time.ZoneId.systemDefault; import static java.util.Arrays.asList; import static java.util.UUID.randomUUID; -import static org.assertj.core.util.Lists.emptyList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -52,7 +43,7 @@ public class DynamoJobRepositoryTest { @BeforeAll public static void initDbs() throws IOException { - testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); } @Container @@ -72,7 +63,7 @@ public void tearDown() { @BeforeEach public void setUp() throws Exception { - testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); } private static DynamoDbClient getDynamoDbClient() { @@ -107,7 +98,7 @@ public void shouldCreateOrUpdateJob() { @Test public void shouldFindJobInfoByUri() { // given - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); // when JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); @@ -119,7 +110,7 @@ public void shouldFindJobInfoByUri() { @Test public void shouldReturnAbsentStatus() { - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); } @@ -174,6 +165,21 @@ public void shouldFindAll() { assertThat(jobInfos.get(1).getJobId(), is("oldest")); } + @Test + public void shouldFindAllWithPaging() { + // given + testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 2); + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest1", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest2", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest3", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final List jobInfos = testee.findAll(); + // then + assertThat(jobInfos.size(), is(5)); + } + @Test public void shouldFindAllinSizeOperation() { // given @@ -188,6 +194,7 @@ public void shouldFindAllinSizeOperation() { @Test public void shouldFindAllinSizeOperationWithPageing() { // given + testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youn44444556gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -197,8 +204,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // when final long count = testee.size(); // then - //TODO write better test with paging enabled in scan - assertThat(count, is(7L)); + assertThat(count, is(6L)); } // From b615356c22de9212bc317ae94344b9802ebcd44b Mon Sep 17 00:00:00 2001 From: Oliver Steenbuck Date: Mon, 11 Nov 2019 09:35:05 +0100 Subject: [PATCH 49/78] add paging to findAll() --- .../dynamo/DynamoJobRepository.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 16319601d..f9a77700a 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -9,10 +9,7 @@ import software.amazon.awssdk.services.dynamodb.model.*; import java.time.OffsetDateTime; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.stream.Collectors; import static java.util.Collections.emptyList; @@ -71,12 +68,22 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { @Override public List findAll() { - ScanRequest findAll = ScanRequest.builder() - .tableName(JOBS_TABLENAME) - .limit(pageSize) - .build(); - final ScanResponse scan = dynamoDbClient.scan(findAll); - return scan.items().stream().map(this::decode).collect(Collectors.toList()); + Map lastKeyEvaluated = null; + List jobs = new ArrayList<>(); + do { + ScanRequest findAll = ScanRequest.builder() + .tableName(JOBS_TABLENAME) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated) + .build(); + + final ScanResponse scan = dynamoDbClient.scan(findAll); + lastKeyEvaluated = scan.lastEvaluatedKey(); + List newJobsFromThisPage = scan.items().stream().map(this::decode).collect(Collectors.toList()); + jobs.addAll(newJobsFromThisPage); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + + return jobs; } @Override From f99dbb29c51d79b590c42729b5c6992775768f23 Mon Sep 17 00:00:00 2001 From: rs/tzm Date: Mon, 11 Nov 2019 15:23:38 +0100 Subject: [PATCH 50/78] =?UTF-8?q?task=20=F0=9F=93=9D|rs/tm|add=20some=20dy?= =?UTF-8?q?namoRepo=20code=20+=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Riko Stave --- .../dynamo/DynamoJobRepository.java | 88 ++++---- .../jobs/repository/dynamo/JobStructure.java | 5 +- .../dynamo/DynamoJobRepositoryTest.java | 195 ++++++++++-------- 3 files changed, 157 insertions(+), 131 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index f9a77700a..1348d9d7c 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -1,29 +1,29 @@ package de.otto.edison.jobs.repository.dynamo; -import com.fasterxml.jackson.databind.ObjectMapper; import de.otto.edison.jobs.domain.JobInfo; import de.otto.edison.jobs.domain.JobMessage; import de.otto.edison.jobs.domain.Level; import de.otto.edison.jobs.repository.JobRepository; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.utils.ImmutableMap; import java.time.OffsetDateTime; import java.util.*; import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.Comparator.comparingLong; +import static java.util.stream.Collectors.*; public class DynamoJobRepository implements JobRepository { public static final String JOBS_TABLENAME = "jobs"; private final DynamoDbClient dynamoDbClient; - private final ObjectMapper objectMapper; private final int pageSize; - public DynamoJobRepository(DynamoDbClient dynamoDbClient, ObjectMapper objectMapper, int pageSize) { + public DynamoJobRepository(DynamoDbClient dynamoDbClient, int pageSize) { this.dynamoDbClient = dynamoDbClient; - this.objectMapper = objectMapper; this.pageSize = pageSize; } @@ -45,7 +45,6 @@ public Optional findOne(String jobId) { } - @Override public List findLatest(int maxCount) { return null; @@ -53,7 +52,15 @@ public List findLatest(int maxCount) { @Override public List findLatestJobsDistinct() { - return null; + return findAll().stream().collect( + groupingBy( + JobInfo::getJobType, + maxBy(comparingLong(jobInfo -> jobInfo.getLastUpdated().toInstant().toEpochMilli())) + )) + .values().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(toList()); } @Override @@ -63,7 +70,26 @@ public List findLatestBy(String type, int maxCount) { @Override public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { - return null; + Map lastKeyEvaluated = null; + List jobs = new ArrayList<>(); + Map expressionAttributeValues = ImmutableMap.of( + ":val", AttributeValue.builder().n(String.valueOf(timeOffset.toInstant().toEpochMilli())).build() + ); + do { + final ScanRequest query = ScanRequest.builder() + .tableName(JOBS_TABLENAME) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated) + .expressionAttributeValues(expressionAttributeValues) + .filterExpression(JobStructure.LAST_UPDATED_EPOCH.key() + " < :val and attribute_not_exists(" + JobStructure.STOPPED.key() + ")") + .build(); + + final ScanResponse response = dynamoDbClient.scan(query); + lastKeyEvaluated = response.lastEvaluatedKey(); + List newJobsFromThisPage = response.items().stream().map(this::decode).collect(Collectors.toList()); + jobs.addAll(newJobsFromThisPage); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + return jobs; } @Override @@ -120,6 +146,7 @@ private Map encode(JobInfo jobInfo) { jobInfo.getStopped().ifPresent(offsetDateTime -> jobAsItem.put(JobStructure.STOPPED.key(), toStringAttributeValue(offsetDateTime))); if (null != jobInfo.getLastUpdated()) { jobAsItem.put(JobStructure.LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); + jobAsItem.put(JobStructure.LAST_UPDATED_EPOCH.key(), toNumberAttributeValue(jobInfo.getLastUpdated().toInstant().toEpochMilli())); } jobAsItem.put(JobStructure.MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); @@ -164,14 +191,17 @@ private JobMessage attributeValueToMessage(AttributeValue attributeValue) { @Override public void removeIfStopped(String jobId) { - - Map keyMap = new HashMap<>(); - keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); - DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() - .tableName(JOBS_TABLENAME) - .key(keyMap) - .build(); - dynamoDbClient.deleteItem(deleteJobRequest); + findOne(jobId).ifPresent(jobInfo -> { + if (jobInfo.isStopped()) { + Map keyMap = new HashMap<>(); + keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); + DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() + .tableName(JOBS_TABLENAME) + .key(keyMap) + .build(); + dynamoDbClient.deleteItem(deleteJobRequest); + } + }); } @Override @@ -227,6 +257,10 @@ private AttributeValue toStringAttributeValue(String value) { return AttributeValue.builder().s(value).build(); } + private AttributeValue toNumberAttributeValue(long value) { + return AttributeValue.builder().n(String.valueOf(value)).build(); + } + private AttributeValue toMapAttributeValue(JobMessage jobMessage) { Map message = new HashMap<>(); message.put(JobStructure.MSG_LEVEL.key(), toStringAttributeValue(jobMessage.getLevel().getKey())); @@ -246,28 +280,4 @@ private AttributeValue toAttributeValueList(List values) { return AttributeValue.builder().l(values).build(); } - public void setupSchema() { - dynamoDbClient.createTable(CreateTableRequest.builder() - .tableName(JOBS_TABLENAME) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName(JobStructure.ID.key()) - .attributeType(ScalarAttributeType.S) - .build()) - .keySchema(KeySchemaElement.builder() - .attributeName(JobStructure.ID.key()) - .keyType(KeyType.HASH) - .build()) - .provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(10L) - .writeCapacityUnits(10L) - .build()) - .build()); - } - - public void deleteTable() { - DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(JOBS_TABLENAME).build(); - dynamoDbClient.deleteTable(deleteTableRequest); - } - } diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java index a67ebecf6..02e56acd0 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java @@ -12,11 +12,12 @@ enum JobStructure { MSG_TEXT("msg"), MSG_LEVEL("level"), HOSTNAME("hostname"), - LAST_UPDATED("lastUpdated"); + LAST_UPDATED("lastUpdated"), + LAST_UPDATED_EPOCH("lastUpdatedEpoch"); private final String key; - private JobStructure(final String key) { + JobStructure(final String key) { this.key = key; } diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 5e01c9639..d6da68ec5 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -1,10 +1,13 @@ package de.otto.edison.jobs.repository.dynamo; -import com.fasterxml.jackson.databind.ObjectMapper; import de.otto.edison.jobs.domain.JobInfo; import de.otto.edison.jobs.domain.JobInfo.JobStatus; import de.otto.edison.jobs.domain.Level; -import org.junit.jupiter.api.*; +import org.hamcrest.Matchers; +import org.hamcrest.collection.IsCollectionWithSize; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -12,11 +15,12 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; -import java.io.IOException; import java.net.URI; import java.time.Clock; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; @@ -24,6 +28,7 @@ import static de.otto.edison.jobs.domain.JobInfo.builder; import static de.otto.edison.jobs.domain.JobInfo.newJobInfo; import static de.otto.edison.jobs.domain.JobMessage.jobMessage; +import static de.otto.edison.jobs.repository.dynamo.DynamoJobRepository.JOBS_TABLENAME; import static de.otto.edison.testsupport.matcher.OptionalMatchers.isAbsent; import static de.otto.edison.testsupport.matcher.OptionalMatchers.isPresent; import static java.time.Clock.fixed; @@ -36,34 +41,44 @@ import static org.hamcrest.Matchers.*; @Testcontainers -public class DynamoJobRepositoryTest { +class DynamoJobRepositoryTest { private static DynamoJobRepository testee; - private static ObjectMapper objectMapper = new ObjectMapper(); - - @BeforeAll - public static void initDbs() throws IOException { - testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); - } @Container - static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + private static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") .withExposedPorts(8000); - @BeforeEach - public void setUpDynamo() { - testee.setupSchema(); + void setUpDynamo() { + + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(JOBS_TABLENAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName(JobStructure.ID.key()) + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName(JobStructure.ID.key()) + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build()); } @AfterEach - public void tearDown() { - testee.deleteTable(); + void tearDown() { + DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() + .tableName(JOBS_TABLENAME).build(); + getDynamoDbClient().deleteTable(deleteTableRequest); } @BeforeEach - public void setUp() throws Exception { - testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); + void setUp() throws Exception { + testee = new DynamoJobRepository(getDynamoDbClient(), 10); } private static DynamoDbClient getDynamoDbClient() { @@ -81,7 +96,7 @@ private static DynamoDbClient getDynamoDbClient() { private Clock clock = systemDefaultZone(); @Test - public void shouldCreateOrUpdateJob() { + void shouldCreateOrUpdateJob() { //Given final JobInfo savedjobInfo = jobInfo("http://localhost/foo", "T_FOO"); testee.createOrUpdate(savedjobInfo); @@ -96,9 +111,9 @@ public void shouldCreateOrUpdateJob() { } @Test - public void shouldFindJobInfoByUri() { + void shouldFindJobInfoByUri() { // given - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), 10); // when JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); @@ -109,33 +124,33 @@ public void shouldFindJobInfoByUri() { } @Test - public void shouldReturnAbsentStatus() { - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 10); + void shouldReturnAbsentStatus() { + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), 10); assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); } @Test - public void shouldNotFailToRemoveMissingJob() { + void shouldNotFailToRemoveMissingJob() { // when testee.removeIfStopped("foo"); // then // no Exception is thrown... } -// @Test -// public void shouldNotRemoveRunningJobs() { -// // given -// final String testUri = "test"; -// testee.createOrUpdate(newJobInfo(testUri, "FOO", systemDefaultZone(), "localhost")); -// // when -// testee.removeIfStopped(testUri); -// // then -// assertThat(testee.size(), is(1L)); + @Test + void shouldNotRemoveRunningJobs() { + // given + final String testUri = "test"; + testee.createOrUpdate(newJobInfo(testUri, "FOO", systemDefaultZone(), "localhost")); + // when + testee.removeIfStopped(testUri); + // then + assertThat(testee.size(), is(1L)); -// } + } @Test - public void shouldRemoveJob() throws Exception { + void shouldRemoveJob() throws Exception { JobInfo stoppedJob = builder() .setJobId("some/job/stopped") .setJobType("test") @@ -153,7 +168,7 @@ public void shouldRemoveJob() throws Exception { } @Test - public void shouldFindAll() { + void shouldFindAll() { // given testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -166,9 +181,9 @@ public void shouldFindAll() { } @Test - public void shouldFindAllWithPaging() { + void shouldFindAllWithPaging() { // given - testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 2); + testee = new DynamoJobRepository(getDynamoDbClient(), 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest1", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -181,7 +196,7 @@ public void shouldFindAllWithPaging() { } @Test - public void shouldFindAllinSizeOperation() { + void shouldFindAllinSizeOperation() { // given testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -192,9 +207,9 @@ public void shouldFindAllinSizeOperation() { } @Test - public void shouldFindAllinSizeOperationWithPageing() { + void shouldFindAllinSizeOperationWithPageing() { // given - testee = new DynamoJobRepository(getDynamoDbClient(), objectMapper, 2); + testee = new DynamoJobRepository(getDynamoDbClient(), 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youn44444556gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -207,49 +222,49 @@ public void shouldFindAllinSizeOperationWithPageing() { assertThat(count, is(6L)); } -// -// @Test -// public void shouldFindLatestDistinct() throws Exception { -// // Given -// Instant now = Instant.now(); -// final JobInfo eins = newJobInfo("eins", "eins", fixed(now.plusSeconds(10), systemDefault()), "localhost"); -// final JobInfo zwei = newJobInfo("zwei", "eins", fixed(now.plusSeconds(20), systemDefault()), "localhost"); -// final JobInfo drei = newJobInfo("drei", "zwei", fixed(now.plusSeconds(30), systemDefault()), "localhost"); -// final JobInfo vier = newJobInfo("vier", "drei", fixed(now.plusSeconds(40), systemDefault()), "localhost"); -// final JobInfo fuenf = newJobInfo("fuenf", "drei", fixed(now.plusSeconds(50), systemDefault()), "localhost"); -// -// testee.createOrUpdate(eins); -// testee.createOrUpdate(zwei); -// testee.createOrUpdate(drei); -// testee.createOrUpdate(vier); -// testee.createOrUpdate(fuenf); -// -// // When -// List latestDistinct = testee.findLatestJobsDistinct(); -// -// // Then -// assertThat(latestDistinct, hasSize(3)); -// assertThat(latestDistinct, Matchers.containsInAnyOrder(fuenf, zwei, drei)); -// } -// -// -// -// @Test -// public void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() throws Exception { -// // given -// testee.createOrUpdate(newJobInfo("deadJob", "FOO", fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); -// testee.createOrUpdate(newJobInfo("running", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); -// -// // when -// final List jobInfos = testee.findRunningWithoutUpdateSince(now().minus(5, ChronoUnit.SECONDS)); -// -// // then -// assertThat(jobInfos, IsCollectionWithSize.hasSize(1)); -// assertThat(jobInfos.get(0).getJobId(), is("deadJob")); -// } -// + + @Test + void shouldFindLatestDistinct() throws Exception { + // Given + Instant now = Instant.now(); + final JobInfo eins = newJobInfo("eins", "eins", fixed(now.plusSeconds(10), systemDefault()), "localhost"); + final JobInfo zwei = newJobInfo("zwei", "eins", fixed(now.plusSeconds(20), systemDefault()), "localhost"); + final JobInfo drei = newJobInfo("drei", "zwei", fixed(now.plusSeconds(30), systemDefault()), "localhost"); + final JobInfo vier = newJobInfo("vier", "drei", fixed(now.plusSeconds(40), systemDefault()), "localhost"); + final JobInfo fuenf = newJobInfo("fuenf", "drei", fixed(now.plusSeconds(50), systemDefault()), "localhost"); + + testee.createOrUpdate(eins); + testee.createOrUpdate(zwei); + testee.createOrUpdate(drei); + testee.createOrUpdate(vier); + testee.createOrUpdate(fuenf); + + // When + List latestDistinct = testee.findLatestJobsDistinct(); + + // Then + assertThat(latestDistinct, hasSize(3)); + assertThat(latestDistinct, Matchers.containsInAnyOrder(fuenf, zwei, drei)); + } + + + + @Test + void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() throws Exception { + // given + testee.createOrUpdate(newJobInfo("deadJob", "FOO", fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("running", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + + // when + final List jobInfos = testee.findRunningWithoutUpdateSince(now().minus(5, ChronoUnit.SECONDS)); + + // then + assertThat(jobInfos, IsCollectionWithSize.hasSize(1)); + assertThat(jobInfos.get(0).getJobId(), is("deadJob")); + } + // @Test -// public void shouldFindLatestByType() { +// void shouldFindLatestByType() { // // given // final String type = "TEST"; // final String otherType = "OTHERTEST"; @@ -269,7 +284,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // } // // @Test -// public void shouldFindLatest() { +// void shouldFindLatest() { // // given // final String type = "TEST"; // final String otherType = "OTHERTEST"; @@ -287,7 +302,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // } // // @Test -// public void shouldFindAllJobsOfSpecificType() throws Exception { +// void shouldFindAllJobsOfSpecificType() throws Exception { // // Given // final String type = "TEST"; // final String otherType = "OTHERTEST"; @@ -315,7 +330,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // } // // @Test -// public void shouldFindStatusOfJob() throws Exception { +// void shouldFindStatusOfJob() throws Exception { // //Given // final String type = "TEST"; // JobInfo jobInfo = newJobInfo("1", type, systemDefaultZone(), "localhost"); @@ -329,7 +344,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // } // // @Test -// public void shouldAppendMessageToJobInfo() throws Exception { +// void shouldAppendMessageToJobInfo() throws Exception { // // String someUri = "someUri"; // OffsetDateTime now = now(); @@ -352,7 +367,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // } // // @Test -// public void shouldUpdateJobStatus() { +// void shouldUpdateJobStatus() { // //Given // final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); //default jobStatus is 'OK' // testee.createOrUpdate(foo); @@ -368,7 +383,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // // // @Test -// public void shouldUpdateJobLastUpdateTime() { +// void shouldUpdateJobLastUpdateTime() { // //Given // final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); // testee.createOrUpdate(foo); @@ -385,7 +400,7 @@ public void shouldFindAllinSizeOperationWithPageing() { // } // // @Test -// public void shouldClearJobInfos() throws Exception { +// void shouldClearJobInfos() throws Exception { // //Given // JobInfo stoppedJob = builder() // .setJobId("some/job/stopped") From b52462afebdd0c008e747d9a26f30dac2115c586 Mon Sep 17 00:00:00 2001 From: tmoeller Date: Mon, 11 Nov 2019 16:43:44 +0100 Subject: [PATCH 51/78] rs/tm|task|add another dynamoJobRepo function --- .../dynamo/DynamoJobRepository.java | 27 ++++++++++++- .../jobs/repository/dynamo/JobStructure.java | 2 +- .../dynamo/DynamoJobRepositoryTest.java | 40 +++++++++---------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 1348d9d7c..ae3538ed8 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -65,7 +65,11 @@ public List findLatestJobsDistinct() { @Override public List findLatestBy(String type, int maxCount) { - return null; + return findByType(type).stream() + .sorted(Comparator.comparingLong(jobInfo -> + jobInfo.getStarted().toInstant().toEpochMilli()).reversed()) + .limit(maxCount) + .collect(Collectors.toList()); } @Override @@ -119,7 +123,26 @@ public List findAllJobInfoWithoutMessages() { @Override public List findByType(String jobType) { - return null; + Map lastKeyEvaluated = null; + List jobs = new ArrayList<>(); + Map expressionAttributeValues = ImmutableMap.of( + ":jobType", AttributeValue.builder().s(jobType).build() + ); + do { + final ScanRequest query = ScanRequest.builder() + .tableName(JOBS_TABLENAME) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated) + .expressionAttributeValues(expressionAttributeValues) + .filterExpression(JobStructure.JOB_TYPE.key() + " = :jobType") + .build(); + + final ScanResponse response = dynamoDbClient.scan(query); + lastKeyEvaluated = response.lastEvaluatedKey(); + List newJobsFromThisPage = response.items().stream().map(this::decode).collect(Collectors.toList()); + jobs.addAll(newJobsFromThisPage); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + return jobs; } @Override diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java index 02e56acd0..eea3cf14b 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java @@ -5,7 +5,7 @@ enum JobStructure { ID("jobId"), STARTED("started"), STOPPED("stopped"), - JOB_TYPE("type"), + JOB_TYPE("jobType"), STATUS("status"), MESSAGES("messages"), MSG_TS("ts"), diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index d6da68ec5..46b1c9010 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -51,7 +51,6 @@ class DynamoJobRepositoryTest { @BeforeEach void setUpDynamo() { - getDynamoDbClient().createTable(CreateTableRequest.builder() .tableName(JOBS_TABLENAME) .attributeDefinitions(AttributeDefinition.builder() @@ -92,7 +91,6 @@ private static DynamoDbClient getDynamoDbClient() { .create(AwsBasicCredentials.create("acc", "sec"))).build(); } - private Clock clock = systemDefaultZone(); @Test @@ -263,25 +261,25 @@ void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() throws Exception { assertThat(jobInfos.get(0).getJobId(), is("deadJob")); } -// @Test -// void shouldFindLatestByType() { -// // given -// final String type = "TEST"; -// final String otherType = "OTHERTEST"; -// -// -// testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); -// testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); -// testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); -// -// // when -// final List jobInfos = testee.findLatestBy(type, 2); -// -// // then -// assertThat(jobInfos.get(0).getJobId(), is("youngest")); -// assertThat(jobInfos.get(1).getJobId(), is("oldest")); -// assertThat(jobInfos, hasSize(2)); -// } + @Test + void shouldFindLatestByType() { + // given + final String type = "TEST"; + final String otherType = "OTHERTEST"; + + + testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); + + // when + final List jobInfos = testee.findLatestBy(type, 2); + + // then + assertThat(jobInfos.get(0).getJobId(), is("youngest")); + assertThat(jobInfos.get(1).getJobId(), is("oldest")); + assertThat(jobInfos, hasSize(2)); + } // // @Test // void shouldFindLatest() { From c71c7f79a12b9a1eb9d2b8a427d8ed65546bd25d Mon Sep 17 00:00:00 2001 From: tmoeller Date: Mon, 11 Nov 2019 17:12:36 +0100 Subject: [PATCH 52/78] rs/tm|task|another dynamo function --- .../dynamo/DynamoJobRepository.java | 16 ++- .../dynamo/DynamoJobRepositoryTest.java | 125 +++++++++--------- 2 files changed, 76 insertions(+), 65 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index ae3538ed8..c092b24bf 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -47,7 +47,10 @@ public Optional findOne(String jobId) { @Override public List findLatest(int maxCount) { - return null; + return findAll().stream() + .sorted(Comparator.comparingLong(jobInfo -> jobInfo.getStarted().toInstant().toEpochMilli()).reversed()) + .limit(maxCount) + .collect(Collectors.toList()); } @Override @@ -77,7 +80,7 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { Map lastKeyEvaluated = null; List jobs = new ArrayList<>(); Map expressionAttributeValues = ImmutableMap.of( - ":val", AttributeValue.builder().n(String.valueOf(timeOffset.toInstant().toEpochMilli())).build() + ":val", AttributeValue.builder().n(String.valueOf(timeOffset.toInstant().toEpochMilli())).build() ); do { final ScanRequest query = ScanRequest.builder() @@ -229,12 +232,19 @@ public void removeIfStopped(String jobId) { @Override public JobInfo.JobStatus findStatus(String jobId) { - return null; + final Optional jobInfo = findOne(jobId); + if (!jobInfo.isPresent()) { + throw new RuntimeException(); + } + return jobInfo.get().getStatus(); } @Override public void appendMessage(String jobId, JobMessage jobMessage) { + findOne(jobId).ifPresent(jobInfo -> { + + }); } @Override diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 46b1c9010..56710a03a 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -2,6 +2,7 @@ import de.otto.edison.jobs.domain.JobInfo; import de.otto.edison.jobs.domain.JobInfo.JobStatus; +import de.otto.edison.jobs.domain.JobMessage; import de.otto.edison.jobs.domain.Level; import org.hamcrest.Matchers; import org.hamcrest.collection.IsCollectionWithSize; @@ -20,6 +21,7 @@ import java.net.URI; import java.time.Clock; import java.time.Instant; +import java.time.OffsetDateTime; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; @@ -280,67 +282,67 @@ void shouldFindLatestByType() { assertThat(jobInfos.get(1).getJobId(), is("oldest")); assertThat(jobInfos, hasSize(2)); } -// -// @Test -// void shouldFindLatest() { -// // given -// final String type = "TEST"; -// final String otherType = "OTHERTEST"; -// testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); -// testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); -// testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); -// -// // when -// final List jobInfos = testee.findLatest(2); -// -// // then -// assertThat(jobInfos.get(0).getJobId(), is("youngest")); -// assertThat(jobInfos.get(1).getJobId(), is("other")); -// assertThat(jobInfos, hasSize(2)); -// } -// -// @Test -// void shouldFindAllJobsOfSpecificType() throws Exception { -// // Given -// final String type = "TEST"; -// final String otherType = "OTHERTEST"; -// testee.createOrUpdate(builder() -// .setJobId("1") -// .setJobType(type) -// .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) -// .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) -// .setHostname("localhost") -// .setStatus(JobStatus.OK) -// .build()); -// testee.createOrUpdate(newJobInfo("2", otherType, systemDefaultZone(), "localhost")); -// testee.createOrUpdate(newJobInfo("3", type, systemDefaultZone(), "localhost")); -// -// // When -// final List jobsType1 = testee.findByType(type); -// final List jobsType2 = testee.findByType(otherType); -// -// // Then -// assertThat(jobsType1.size(), is(2)); -// assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("1")), is(true)); -// assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("3")), is(true)); -// assertThat(jobsType2.size(), is(1)); -// assertThat(jobsType2.stream().anyMatch(job -> job.getJobId().equals("2")), is(true)); -// } -// -// @Test -// void shouldFindStatusOfJob() throws Exception { -// //Given -// final String type = "TEST"; -// JobInfo jobInfo = newJobInfo("1", type, systemDefaultZone(), "localhost"); -// testee.createOrUpdate(jobInfo); -// -// //When -// JobStatus status = testee.findStatus("1"); -// -// //Then -// assertThat(status, is(JobStatus.OK)); -// } -// + + @Test + void shouldFindLatest() { + // given + final String type = "TEST"; + final String otherType = "OTHERTEST"; + testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); + + // when + final List jobInfos = testee.findLatest(2); + + // then + assertThat(jobInfos.get(0).getJobId(), is("youngest")); + assertThat(jobInfos.get(1).getJobId(), is("other")); + assertThat(jobInfos, hasSize(2)); + } + + @Test + void shouldFindAllJobsOfSpecificType() throws Exception { + // Given + final String type = "TEST"; + final String otherType = "OTHERTEST"; + testee.createOrUpdate(builder() + .setJobId("1") + .setJobType(type) + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build()); + testee.createOrUpdate(newJobInfo("2", otherType, systemDefaultZone(), "localhost")); + testee.createOrUpdate(newJobInfo("3", type, systemDefaultZone(), "localhost")); + + // When + final List jobsType1 = testee.findByType(type); + final List jobsType2 = testee.findByType(otherType); + + // Then + assertThat(jobsType1.size(), is(2)); + assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("1")), is(true)); + assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("3")), is(true)); + assertThat(jobsType2.size(), is(1)); + assertThat(jobsType2.stream().anyMatch(job -> job.getJobId().equals("2")), is(true)); + } + + @Test + void shouldFindStatusOfJob() throws Exception { + //Given + final String type = "TEST"; + JobInfo jobInfo = newJobInfo("1", type, systemDefaultZone(), "localhost"); + testee.createOrUpdate(jobInfo); + + //When + JobStatus status = testee.findStatus("1"); + + //Then + assertThat(status, is(JobStatus.OK)); + } + // @Test // void shouldAppendMessageToJobInfo() throws Exception { // @@ -361,7 +363,6 @@ void shouldFindLatestByType() { // assertThat(jobInfoFromRepo.getMessages().size(), is(1)); // assertThat(jobInfoFromRepo.getMessages().get(0), is(igelMessage)); // assertThat(jobInfoFromRepo.getLastUpdated(), is(now.truncatedTo(ChronoUnit.MILLIS))); -// // } // // @Test From 4abfc8c4453dd807f5e3391c7d7666de6664be18 Mon Sep 17 00:00:00 2001 From: tmoeller Date: Tue, 12 Nov 2019 10:42:14 +0100 Subject: [PATCH 53/78] rf/tm|task|add last dynamoRepo functions --- .../dynamo/DynamoJobMetaRepository.java | 27 ++- .../dynamo/DynamoJobRepository.java | 68 +++++--- .../dynamo/DynamoJobRepositoryTest.java | 156 +++++++++--------- .../mongo/JobMetaRepositoryTest.java | 22 ++- 4 files changed, 161 insertions(+), 112 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index 2863344e9..5dd21339e 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -17,6 +17,7 @@ public class DynamoJobMetaRepository implements JobMetaRepository { private static final String KEY_DISABLED = "_e_disabled"; private static final String KEY_RUNNING = "_e_running"; + private static final String JOB_META_TABLE_NAME = "jobMeta"; private final DynamoDbClient dynamoDbClient; public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { @@ -27,7 +28,7 @@ public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { public JobMeta getJobMeta(String jobType) { ImmutableMap key = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .key(key) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); @@ -97,7 +98,7 @@ public String setValue(String jobType, String key, String value) { private String putValue(String jobType, String key, String value) { ImmutableMap itemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .key(itemRequestKey) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); @@ -117,7 +118,7 @@ private String putValue(String jobType, String key, String value) { } PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .item(newEntry) .build(); dynamoDbClient.putItem(putItemRequest); @@ -132,7 +133,7 @@ private AttributeValue toAttributeValue(String value) { private void putIfAbsent(String jobType) { ImmutableMap itemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .key(itemRequestKey) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); @@ -140,7 +141,7 @@ private void putIfAbsent(String jobType) { Map item = new HashMap<>(); item.put("jobType", toAttributeValue(jobType)); PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .item(item) .build(); dynamoDbClient.putItem(putItemRequest); @@ -151,7 +152,7 @@ private void putIfAbsent(String jobType) { public String getValue(String jobType, String key) { ImmutableMap getItemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .key(getItemRequestKey) .build(); GetItemResponse response = dynamoDbClient.getItem(itemRequest); @@ -166,7 +167,7 @@ public String getValue(String jobType, String key) { @Override public Set findAllJobTypes() { ScanRequest scanRequest = ScanRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .attributesToGet("jobType").build(); ScanResponse scanResponse = dynamoDbClient.scan(scanRequest); Set jobTypes = scanResponse.items().stream().map(m -> m.get("jobType").s()).collect(Collectors.toSet()); @@ -176,12 +177,12 @@ public Set findAllJobTypes() { @Override public void deleteAll() { deleteTable(); - setupSchema(); + createTable(); } - public void setupSchema() { + private void createTable() { dynamoDbClient.createTable(CreateTableRequest.builder() - .tableName("jobMeta") + .tableName(JOB_META_TABLE_NAME) .attributeDefinitions(AttributeDefinition.builder() .attributeName("jobType") .attributeType(ScalarAttributeType.S) @@ -197,11 +198,9 @@ public void setupSchema() { .build()); } - public void deleteTable() { + private void deleteTable() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName("jobMeta").build(); + .tableName(JOB_META_TABLE_NAME).build(); dynamoDbClient.deleteTable(deleteTableRequest); } - - } diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index c092b24bf..db35ead79 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -18,7 +18,7 @@ public class DynamoJobRepository implements JobRepository { - public static final String JOBS_TABLENAME = "jobs"; + private static final String JOBS_TABLE_NAME = "jobs"; private final DynamoDbClient dynamoDbClient; private final int pageSize; @@ -33,7 +33,7 @@ public Optional findOne(String jobId) { keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); GetItemRequest jobInfoRequest = GetItemRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .key(keyMap) .build(); final GetItemResponse jobInfoResponse = dynamoDbClient.getItem(jobInfoRequest); @@ -84,7 +84,7 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { ); do { final ScanRequest query = ScanRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .expressionAttributeValues(expressionAttributeValues) @@ -105,7 +105,7 @@ public List findAll() { List jobs = new ArrayList<>(); do { ScanRequest findAll = ScanRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .build(); @@ -133,7 +133,7 @@ public List findByType(String jobType) { ); do { final ScanRequest query = ScanRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .expressionAttributeValues(expressionAttributeValues) @@ -153,7 +153,7 @@ public JobInfo createOrUpdate(JobInfo job) { Map jobAsItem = encode(job); PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .item(jobAsItem) .build(); dynamoDbClient.putItem(putItemRequest); @@ -222,7 +222,7 @@ public void removeIfStopped(String jobId) { Map keyMap = new HashMap<>(); keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .key(keyMap) .build(); dynamoDbClient.deleteItem(deleteJobRequest); @@ -232,29 +232,34 @@ public void removeIfStopped(String jobId) { @Override public JobInfo.JobStatus findStatus(String jobId) { - final Optional jobInfo = findOne(jobId); - if (!jobInfo.isPresent()) { - throw new RuntimeException(); - } - return jobInfo.get().getStatus(); + return findOne(jobId) + .orElseThrow(RuntimeException::new) + .getStatus(); } @Override public void appendMessage(String jobId, JobMessage jobMessage) { - findOne(jobId).ifPresent(jobInfo -> { - - - }); + JobInfo jobInfo = findOne(jobId).orElseThrow(RuntimeException::new); + createOrUpdate(jobInfo.copy() + .addMessage(jobMessage) + .setLastUpdated(jobMessage.getTimestamp()) + .build()); } @Override public void setJobStatus(String jobId, JobInfo.JobStatus jobStatus) { - + JobInfo jobInfo = findOne(jobId).orElseThrow(RuntimeException::new); + createOrUpdate(jobInfo.copy() + .setStatus(jobStatus) + .build()); } @Override public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { - + JobInfo jobInfo = findOne(jobId).orElseThrow(RuntimeException::new); + createOrUpdate(jobInfo.copy() + .setLastUpdated(lastUpdate) + .build()); } @Override @@ -263,7 +268,7 @@ public long size() { long count = 0; do { ScanRequest counterQuery = ScanRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .select(Select.COUNT) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) @@ -279,7 +284,32 @@ public long size() { @Override public void deleteAll() { + deleteTable(); + createTable(); + } + + private void deleteTable() { + DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() + .tableName(JOBS_TABLE_NAME).build(); + dynamoDbClient.deleteTable(deleteTableRequest); + } + private void createTable() { + dynamoDbClient.createTable(CreateTableRequest.builder() + .tableName(JOBS_TABLE_NAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName(JobStructure.ID.key()) + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName(JobStructure.ID.key()) + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build()); } private AttributeValue toStringAttributeValue(OffsetDateTime value) { diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 56710a03a..145db59d4 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -22,15 +22,16 @@ import java.time.Clock; import java.time.Instant; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; +import static de.otto.edison.jobs.domain.JobInfo.JobStatus.ERROR; import static de.otto.edison.jobs.domain.JobInfo.JobStatus.OK; import static de.otto.edison.jobs.domain.JobInfo.builder; import static de.otto.edison.jobs.domain.JobInfo.newJobInfo; import static de.otto.edison.jobs.domain.JobMessage.jobMessage; -import static de.otto.edison.jobs.repository.dynamo.DynamoJobRepository.JOBS_TABLENAME; import static de.otto.edison.testsupport.matcher.OptionalMatchers.isAbsent; import static de.otto.edison.testsupport.matcher.OptionalMatchers.isPresent; import static java.time.Clock.fixed; @@ -38,6 +39,7 @@ import static java.time.OffsetDateTime.now; import static java.time.ZoneId.systemDefault; import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; import static java.util.UUID.randomUUID; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; @@ -45,6 +47,8 @@ @Testcontainers class DynamoJobRepositoryTest { + private static final String JOBS_TABLE_NAME = "jobs"; + private static DynamoJobRepository testee; @Container @@ -54,7 +58,7 @@ class DynamoJobRepositoryTest { @BeforeEach void setUpDynamo() { getDynamoDbClient().createTable(CreateTableRequest.builder() - .tableName(JOBS_TABLENAME) + .tableName(JOBS_TABLE_NAME) .attributeDefinitions(AttributeDefinition.builder() .attributeName(JobStructure.ID.key()) .attributeType(ScalarAttributeType.S) @@ -73,7 +77,7 @@ void setUpDynamo() { @AfterEach void tearDown() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(JOBS_TABLENAME).build(); + .tableName(JOBS_TABLE_NAME).build(); getDynamoDbClient().deleteTable(deleteTableRequest); } @@ -343,80 +347,78 @@ void shouldFindStatusOfJob() throws Exception { assertThat(status, is(JobStatus.OK)); } -// @Test -// void shouldAppendMessageToJobInfo() throws Exception { -// -// String someUri = "someUri"; -// OffsetDateTime now = now(); -// -// //Given -// JobInfo jobInfo = newJobInfo(someUri, "TEST", systemDefaultZone(), "localhost"); -// testee.createOrUpdate(jobInfo); -// -// //When -// JobMessage igelMessage = JobMessage.jobMessage(Level.WARNING, "Der Igel ist froh.", now); -// testee.appendMessage(someUri, igelMessage); -// -// //Then -// JobInfo jobInfoFromRepo = testee.findOne(someUri).get(); -// -// assertThat(jobInfoFromRepo.getMessages().size(), is(1)); -// assertThat(jobInfoFromRepo.getMessages().get(0), is(igelMessage)); -// assertThat(jobInfoFromRepo.getLastUpdated(), is(now.truncatedTo(ChronoUnit.MILLIS))); -// } -// -// @Test -// void shouldUpdateJobStatus() { -// //Given -// final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); //default jobStatus is 'OK' -// testee.createOrUpdate(foo); -// -// //When -// testee.setJobStatus(foo.getJobId(), ERROR); -// JobStatus status = testee.findStatus("http://localhost/foo"); -// -// //Then -// assertThat(status, is(ERROR)); -// } -// -// -// -// @Test -// void shouldUpdateJobLastUpdateTime() { -// //Given -// final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); -// testee.createOrUpdate(foo); -// -// OffsetDateTime myTestTime = OffsetDateTime.of(1979, 2, 5, 1, 2, 3, 1_000_000, ZoneOffset.UTC); -// -// //When -// testee.setLastUpdate(foo.getJobId(), myTestTime); -// -// final Optional jobInfo = testee.findOne(foo.getJobId()); -// -// //Then -// assertThat(jobInfo.get().getLastUpdated(), is(myTestTime)); -// } -// -// @Test -// void shouldClearJobInfos() throws Exception { -// //Given -// JobInfo stoppedJob = builder() -// .setJobId("some/job/stopped") -// .setJobType("test") -// .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) -// .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) -// .setHostname("localhost") -// .setStatus(JobStatus.OK) -// .build(); -// testee.createOrUpdate(stoppedJob); -// -// //When -// testee.deleteAll(); -// -// //Then -// assertThat(testee.findAll(), is(emptyList())); -// } + @Test + void shouldAppendMessageToJobInfo() throws Exception { + + String someUri = "someUri"; + OffsetDateTime now = now(); + + //Given + JobInfo jobInfo = newJobInfo(someUri, "TEST", systemDefaultZone(), "localhost"); + testee.createOrUpdate(jobInfo); + + //When + JobMessage igelMessage = JobMessage.jobMessage(Level.WARNING, "Der Igel ist froh.", now); + testee.appendMessage(someUri, igelMessage); + + //Then + JobInfo jobInfoFromRepo = testee.findOne(someUri).get(); + + assertThat(jobInfoFromRepo.getMessages().size(), is(1)); + assertThat(jobInfoFromRepo.getMessages().get(0), is(igelMessage)); + assertThat(jobInfoFromRepo.getLastUpdated(), is(now.truncatedTo(ChronoUnit.MILLIS))); + } + + @Test + void shouldUpdateJobStatus() { + //Given + final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); //default jobStatus is 'OK' + testee.createOrUpdate(foo); + + //When + testee.setJobStatus(foo.getJobId(), ERROR); + JobStatus status = testee.findStatus("http://localhost/foo"); + + //Then + assertThat(status, is(ERROR)); + } + + @Test + void shouldUpdateJobLastUpdateTime() { + //Given + final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); + testee.createOrUpdate(foo); + + OffsetDateTime myTestTime = OffsetDateTime.of(1979, 2, 5, 1, 2, 3, 1_000_000, ZoneOffset.UTC); + + //When + testee.setLastUpdate(foo.getJobId(), myTestTime); + + final Optional jobInfo = testee.findOne(foo.getJobId()); + + //Then + assertThat(jobInfo.get().getLastUpdated(), is(myTestTime)); + } + + @Test + void shouldClearJobInfos() throws Exception { + //Given + JobInfo stoppedJob = builder() + .setJobId("some/job/stopped") + .setJobType("test") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build(); + testee.createOrUpdate(stoppedJob); + + //When + testee.deleteAll(); + + //Then + assertThat(testee.findAll(), is(emptyList())); + } private JobInfo jobInfo(final String jobId, final String type) { return JobInfo.newJobInfo( diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index a356796d5..70d5f5cfd 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -19,6 +19,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; import java.io.IOException; import java.net.URI; @@ -35,6 +36,7 @@ @Testcontainers public class JobMetaRepositoryTest { + private static final String JOB_META_TABLE_NAME = "jobMeta"; private static DynamoJobMetaRepository dynamoTestee = null; @AfterAll @@ -56,12 +58,28 @@ public static void initDbs() throws IOException { @BeforeEach public void setUpDynamo() { - dynamoTestee.setupSchema(); + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(JOB_META_TABLE_NAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName("jobType") + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName("jobType") + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build());; } @AfterEach public void tearDown() { - dynamoTestee.deleteTable(); + DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() + .tableName(JOB_META_TABLE_NAME).build(); + getDynamoDbClient().deleteTable(deleteTableRequest); } public static Collection data() { From e061453f97f5274c7a91711803237623cd2ada5d Mon Sep 17 00:00:00 2001 From: tmoeller Date: Tue, 12 Nov 2019 12:53:37 +0100 Subject: [PATCH 54/78] rf/tm|task|create table for dynamo(meta)repos at startup --- .../dynamo/DynamoJobMetaRepository.java | 9 +- .../dynamo/DynamoJobRepository.java | 9 +- .../dynamo/DynamoJobRepositoryTest.java | 39 ++---- .../mongo/JobMetaRepositoryTest.java | 121 ++++++++++-------- 4 files changed, 93 insertions(+), 85 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index 5dd21339e..3e92f577d 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -17,11 +17,18 @@ public class DynamoJobMetaRepository implements JobMetaRepository { private static final String KEY_DISABLED = "_e_disabled"; private static final String KEY_RUNNING = "_e_running"; - private static final String JOB_META_TABLE_NAME = "jobMeta"; + private static final String JOB_META_TABLE_NAME = "FT6_DynamoDB_JobMeta"; private final DynamoDbClient dynamoDbClient; public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { this.dynamoDbClient = dynamoDbClient; + try { + dynamoDbClient.describeTable(DescribeTableRequest.builder() + .tableName(JOB_META_TABLE_NAME) + .build()); + } catch (ResourceNotFoundException e) { + createTable(); + } } @Override diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index db35ead79..34033e9ff 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -18,13 +18,20 @@ public class DynamoJobRepository implements JobRepository { - private static final String JOBS_TABLE_NAME = "jobs"; + private static final String JOBS_TABLE_NAME = "FT6_DynamoDB_Jobs"; private final DynamoDbClient dynamoDbClient; private final int pageSize; public DynamoJobRepository(DynamoDbClient dynamoDbClient, int pageSize) { this.dynamoDbClient = dynamoDbClient; this.pageSize = pageSize; + try { + dynamoDbClient.describeTable(DescribeTableRequest.builder() + .tableName(JOBS_TABLE_NAME) + .build()); + } catch (ResourceNotFoundException e) { + createTable(); + } } @Override diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 145db59d4..0ecbe5114 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -16,7 +16,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; import java.net.URI; import java.time.Clock; @@ -47,7 +47,7 @@ @Testcontainers class DynamoJobRepositoryTest { - private static final String JOBS_TABLE_NAME = "jobs"; + private static final String JOBS_TABLE_NAME = "FT6_DynamoDB_Jobs"; private static DynamoJobRepository testee; @@ -55,25 +55,6 @@ class DynamoJobRepositoryTest { private static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") .withExposedPorts(8000); - @BeforeEach - void setUpDynamo() { - getDynamoDbClient().createTable(CreateTableRequest.builder() - .tableName(JOBS_TABLE_NAME) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName(JobStructure.ID.key()) - .attributeType(ScalarAttributeType.S) - .build()) - .keySchema(KeySchemaElement.builder() - .attributeName(JobStructure.ID.key()) - .keyType(KeyType.HASH) - .build()) - .provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(10L) - .writeCapacityUnits(10L) - .build()) - .build()); - } - @AfterEach void tearDown() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() @@ -82,7 +63,7 @@ void tearDown() { } @BeforeEach - void setUp() throws Exception { + void setUp() { testee = new DynamoJobRepository(getDynamoDbClient(), 10); } @@ -154,7 +135,7 @@ void shouldNotRemoveRunningJobs() { } @Test - void shouldRemoveJob() throws Exception { + void shouldRemoveJob() { JobInfo stoppedJob = builder() .setJobId("some/job/stopped") .setJobType("test") @@ -228,7 +209,7 @@ void shouldFindAllinSizeOperationWithPageing() { @Test - void shouldFindLatestDistinct() throws Exception { + void shouldFindLatestDistinct() { // Given Instant now = Instant.now(); final JobInfo eins = newJobInfo("eins", "eins", fixed(now.plusSeconds(10), systemDefault()), "localhost"); @@ -254,7 +235,7 @@ void shouldFindLatestDistinct() throws Exception { @Test - void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() throws Exception { + void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() { // given testee.createOrUpdate(newJobInfo("deadJob", "FOO", fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("running", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -306,7 +287,7 @@ void shouldFindLatest() { } @Test - void shouldFindAllJobsOfSpecificType() throws Exception { + void shouldFindAllJobsOfSpecificType() { // Given final String type = "TEST"; final String otherType = "OTHERTEST"; @@ -334,7 +315,7 @@ void shouldFindAllJobsOfSpecificType() throws Exception { } @Test - void shouldFindStatusOfJob() throws Exception { + void shouldFindStatusOfJob() { //Given final String type = "TEST"; JobInfo jobInfo = newJobInfo("1", type, systemDefaultZone(), "localhost"); @@ -348,7 +329,7 @@ void shouldFindStatusOfJob() throws Exception { } @Test - void shouldAppendMessageToJobInfo() throws Exception { + void shouldAppendMessageToJobInfo() { String someUri = "someUri"; OffsetDateTime now = now(); @@ -401,7 +382,7 @@ void shouldUpdateJobLastUpdateTime() { } @Test - void shouldClearJobInfos() throws Exception { + void shouldClearJobInfos() { //Given JobInfo stoppedJob = builder() .setJobId("some/job/stopped") diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index 70d5f5cfd..8c3d543b3 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -19,11 +19,12 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; import java.io.IOException; import java.net.URI; import java.util.Collection; +import java.util.Set; import java.util.UUID; import static java.util.Arrays.asList; @@ -36,7 +37,7 @@ @Testcontainers public class JobMetaRepositoryTest { - private static final String JOB_META_TABLE_NAME = "jobMeta"; + private static final String DYNAMO_JOB_META_TABLE_NAME = "FT6_DynamoDB_JobMeta"; private static DynamoJobMetaRepository dynamoTestee = null; @AfterAll @@ -52,38 +53,21 @@ public static void initDbs() throws IOException { @Container static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") - .withExposedPorts(8000); - - String dynamoTableName = "jobMeta"; + .withExposedPorts(8000);; @BeforeEach - public void setUpDynamo() { - getDynamoDbClient().createTable(CreateTableRequest.builder() - .tableName(JOB_META_TABLE_NAME) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName("jobType") - .attributeType(ScalarAttributeType.S) - .build()) - .keySchema(KeySchemaElement.builder() - .attributeName("jobType") - .keyType(KeyType.HASH) - .build()) - .provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(10L) - .writeCapacityUnits(10L) - .build()) - .build());; + void setUp() { + dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient()); } @AfterEach public void tearDown() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(JOB_META_TABLE_NAME).build(); + .tableName(DYNAMO_JOB_META_TABLE_NAME).build(); getDynamoDbClient().deleteTable(deleteTableRequest); } - public static Collection data() { - DynamoDbClient dynamoBuilder = getDynamoDbClient(); + private static Collection data() { return asList( new MongoJobMetaRepository(EmbeddedMongoHelper.getMongoClient().getDatabase("jobmeta-" + UUID.randomUUID()), "jobmeta", @@ -96,7 +80,6 @@ public static Collection data() { private static DynamoDbClient getDynamoDbClient() { String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + dynamodb.getMappedPort(8000); - return DynamoDbClient.builder() .endpointOverride(URI.create(endpointUri)) .region(Region.EU_CENTRAL_1) @@ -107,12 +90,15 @@ private static DynamoDbClient getDynamoDbClient() { @ParameterizedTest @MethodSource("data") public void shouldStoreAndGetValue(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when testee.setValue("someJob", "someKey", "someValue"); testee.setValue("someJob", "someOtherKey", "someDifferentValue"); - testee.setValue("someOtherJob", "someKey", "someOtherValue"); + //then assertThat(testee.getValue("someJob", "someKey"), is("someValue")); assertThat(testee.getValue("someJob", "someOtherKey"), is("someDifferentValue")); assertThat(testee.getValue("someOtherJob", "someKey"), is("someOtherValue")); @@ -121,9 +107,13 @@ public void shouldStoreAndGetValue(final JobMetaRepository testee) { @ParameterizedTest @MethodSource("data") public void shouldGetEmptyJobMeta(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(emptyMap())); assertThat(jobMeta.isDisabled(), is(false)); assertThat(jobMeta.getDisabledComment(), is("")); @@ -131,14 +121,17 @@ public void shouldGetEmptyJobMeta(final JobMetaRepository testee) { assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldGetJobMetaForRunningJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setRunningJob("someJob", "someId"); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(emptyMap())); assertThat(jobMeta.isDisabled(), is(false)); assertThat(jobMeta.getDisabledComment(), is("")); @@ -146,14 +139,17 @@ public void shouldGetJobMetaForRunningJob(final JobMetaRepository testee) { assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldGetJobMetaForDisabledJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.disable("someJob", "some comment"); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(emptyMap())); assertThat(jobMeta.isDisabled(), is(true)); assertThat(jobMeta.getDisabledComment(), is("some comment")); @@ -161,15 +157,18 @@ public void shouldGetJobMetaForDisabledJob(final JobMetaRepository testee) { assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldGetJobMetaForDisabledJobWithProperties(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.disable("someJob", "some comment"); testee.setValue("someJob", "someKey", "some value"); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(singletonMap("someKey", "some value"))); assertThat(jobMeta.isDisabled(), is(true)); assertThat(jobMeta.getDisabledComment(), is("some comment")); @@ -177,35 +176,43 @@ public void shouldGetJobMetaForDisabledJobWithProperties(final JobMetaRepository assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldEnableJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "_e_disabled", "foo"); + //when testee.enable("someJob"); + //then assertThat(testee.getValue("someJob", "_e_disabled"), is(nullValue())); } - @ParameterizedTest @MethodSource("data") public void shouldDisableJob(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when testee.disable("someJob", "some comment"); + //then assertThat(testee.getValue("someJob", "_e_disabled"), is("some comment")); } - @ParameterizedTest @MethodSource("data") public void shouldSetRunningJob(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when testee.setRunningJob("someJob", "someId"); + //then assertThat(testee.getRunningJob("someJob"), is("someId")); assertThat(testee.getValue("someJob", "_e_running"), is("someId")); } @@ -213,59 +220,68 @@ public void shouldSetRunningJob(final JobMetaRepository testee) { @ParameterizedTest @MethodSource("data") public void shouldDeleteAll(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.enable("foo"); testee.enable("bar"); + //when testee.deleteAll(); + //then assertThat(testee.findAllJobTypes(), is(empty())); } - @ParameterizedTest @MethodSource("data") public void shouldClearRunningJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "_e_running", "someId"); + //when testee.clearRunningJob("someJob"); + //then assertThat(testee.getRunningJob("someJob"), is(nullValue())); assertThat(testee.getValue("someJob", "_e_runnin"), is(nullValue())); } - @ParameterizedTest @MethodSource("data") public void shouldReturnNullForMissingKeys(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "someKey", "someValue"); + //when/then assertThat(testee.getValue("someJob", "someMissingKey"), is(nullValue())); assertThat(testee.getValue("someMissingJob", "someMissingKey"), is(nullValue())); } - @ParameterizedTest @MethodSource("data") public void shouldFindJobTypes(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "someKey", "someValue"); testee.setValue("someOtherJob", "someKey", "someOtherValue"); - assertThat(testee.findAllJobTypes(), containsInAnyOrder("someJob", "someOtherJob")); + //when + final Set allJobTypes = testee.findAllJobTypes(); + + //then + assertThat(allJobTypes, containsInAnyOrder("someJob", "someOtherJob")); } - @ParameterizedTest @MethodSource("data") public void shouldNotCreateIfExists(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.setValue("someJob", "someKey", "initialValue"); - // when + //when final boolean value = testee.createValue("someJob", "someKey", "newValue"); //then @@ -273,12 +289,13 @@ public void shouldNotCreateIfExists(final JobMetaRepository testee) { assertThat(testee.getValue("someJob", "someKey"), is("initialValue")); } - @ParameterizedTest @MethodSource("data") public void shouldCreateIfNotExists(final JobMetaRepository testee) { + //given testee.deleteAll(); - // when + + //when final boolean value = testee.createValue("someJob", "someKey", "someValue"); //then @@ -286,15 +303,14 @@ public void shouldCreateIfNotExists(final JobMetaRepository testee) { assertThat(testee.getValue("someJob", "someKey"), is("someValue")); } - @ParameterizedTest @MethodSource("data") public void shouldCreateTwoValuesWithoutException(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.createValue("someJob", "someKey", "someValue"); - // when + //when final boolean value = testee.createValue("someJob", "someOtherKey", "someOtherValue"); //then @@ -303,15 +319,14 @@ public void shouldCreateTwoValuesWithoutException(final JobMetaRepository testee assertThat(testee.getValue("someJob", "someOtherKey"), is("someOtherValue")); } - @ParameterizedTest @MethodSource("data") public void shouldReturnFalseIfCreateWasCalledTwice(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.createValue("someJob", "someKey", "someInitialValue"); - // when + //when final boolean value = testee.createValue("someJob", "someKey", "someValue"); //then @@ -319,15 +334,14 @@ public void shouldReturnFalseIfCreateWasCalledTwice(final JobMetaRepository test assertThat(testee.getValue("someJob", "someKey"), is("someInitialValue")); } - @ParameterizedTest @MethodSource("data") public void shouldNotKillOldFieldsOnCreate(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.setValue("someJob", "someKey", "someInitialValue"); - // when + //when final boolean value = testee.createValue("someJob", "someAtomicKey", "someValue"); //then @@ -336,18 +350,17 @@ public void shouldNotKillOldFieldsOnCreate(final JobMetaRepository testee) { assertThat(testee.getValue("someJob", "someAtomicKey"), is("someValue")); } - @ParameterizedTest @MethodSource("data") public void shouldUnsetKeyOnSetNullValue(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.setValue("someJob", "someKey", "someValue"); - // when + //when testee.setValue("someJob", "someKey", null); - // then + //then assertThat(testee.findAllJobTypes(), contains("someJob")); assertThat(testee.getValue("someJob", "someKey"), is(nullValue())); } From 676996724d25a7f0c0e8faea22c9e45ebdc7a25e Mon Sep 17 00:00:00 2001 From: tmoeller Date: Tue, 12 Nov 2019 17:08:12 +0100 Subject: [PATCH 55/78] rf/tm|task|add transactional stuff to DynamoMetaRepo --- .../dynamo/DynamoJobMetaRepository.java | 96 ++++++++++--------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index 3e92f577d..c1e56537b 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -9,15 +9,19 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static java.util.Collections.emptyMap; public class DynamoJobMetaRepository implements JobMetaRepository { - private static final String KEY_DISABLED = "_e_disabled"; - private static final String KEY_RUNNING = "_e_running"; + private static final String KEY_PREFIX = "_e_"; + private static final String KEY_DISABLED = KEY_PREFIX + "disabled"; + private static final String KEY_RUNNING = KEY_PREFIX + "running"; private static final String JOB_META_TABLE_NAME = "FT6_DynamoDB_JobMeta"; + private static final String JOB_TYPE_KEY = "jobType"; + private static final String ETAG_KEY = "etag"; private final DynamoDbClient dynamoDbClient; public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { @@ -33,12 +37,7 @@ public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { @Override public JobMeta getJobMeta(String jobType) { - ImmutableMap key = ImmutableMap.of("jobType", toAttributeValue(jobType)); - GetItemRequest itemRequest = GetItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) - .key(key) - .build(); - GetItemResponse response = dynamoDbClient.getItem(itemRequest); + GetItemResponse response = getItem(jobType); Map responseItem = response.item(); if (!responseItem.isEmpty()) { @@ -51,8 +50,9 @@ public JobMeta getJobMeta(String jobType) { } Map metaMap = responseItem.entrySet().stream() - .filter(e -> !e.getKey().startsWith("_e_")) - .filter(e -> !e.getKey().equals("jobType")) + .filter(e -> !e.getKey().startsWith(KEY_PREFIX)) + .filter(e -> !e.getKey().equals(JOB_TYPE_KEY)) + .filter(e -> !e.getKey().equals(ETAG_KEY)) .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().s())); return new JobMeta(jobType, isRunning, isDisabled, comment, metaMap); } else { @@ -103,34 +103,39 @@ public String setValue(String jobType, String key, String value) { } private String putValue(String jobType, String key, String value) { - ImmutableMap itemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); - GetItemRequest itemRequest = GetItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) - .key(itemRequestKey) - .build(); - GetItemResponse response = dynamoDbClient.getItem(itemRequest); - Map newEntry = new HashMap<>(); + GetItemResponse getItemResponse = getItem(jobType); - AttributeValue previousAttributeValue = response.item().get(key); - String previous = null; - if (previousAttributeValue != null) { - previous = previousAttributeValue.s(); - } - - newEntry.putAll(response.item()); + Map newEntry = new HashMap<>(getItemResponse.item()); if (value == null) { newEntry.remove(key); } else { newEntry.put(key, toAttributeValue(value)); } + newEntry.put(ETAG_KEY, AttributeValue.builder().s(UUID.randomUUID().toString()).build()); - PutItemRequest putItemRequest = PutItemRequest.builder() + final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() .tableName(JOB_META_TABLE_NAME) - .item(newEntry) - .build(); - dynamoDbClient.putItem(putItemRequest); + .item(newEntry); + + addEtagCondition(putItemRequestBuilder, getItemResponse); + + dynamoDbClient.putItem(putItemRequestBuilder.build()); + + AttributeValue existingValueForKey = getItemResponse.item().get(key); + if (existingValueForKey == null) { + return null; + } + return existingValueForKey.s(); + } - return previous; + private void addEtagCondition(final PutItemRequest.Builder putItemRequestBuilder, final GetItemResponse getItemResponse) { + final AttributeValue existingEtag = getItemResponse.item().get(ETAG_KEY); + if (existingEtag != null) { + Map valueMap = new HashMap<>(); + valueMap.put(":val", AttributeValue.builder().s(existingEtag.s()).build()); + putItemRequestBuilder.expressionAttributeValues(valueMap); + putItemRequestBuilder.conditionExpression("contains(etag, :val)"); + } } private AttributeValue toAttributeValue(String value) { @@ -138,15 +143,10 @@ private AttributeValue toAttributeValue(String value) { } private void putIfAbsent(String jobType) { - ImmutableMap itemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); - GetItemRequest itemRequest = GetItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) - .key(itemRequestKey) - .build(); - GetItemResponse response = dynamoDbClient.getItem(itemRequest); + GetItemResponse response = getItem(jobType); if (response.item().isEmpty()) { Map item = new HashMap<>(); - item.put("jobType", toAttributeValue(jobType)); + item.put(JOB_TYPE_KEY, toAttributeValue(jobType)); PutItemRequest putItemRequest = PutItemRequest.builder() .tableName(JOB_META_TABLE_NAME) .item(item) @@ -157,12 +157,7 @@ private void putIfAbsent(String jobType) { @Override public String getValue(String jobType, String key) { - ImmutableMap getItemRequestKey = ImmutableMap.of("jobType", toAttributeValue(jobType)); - GetItemRequest itemRequest = GetItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) - .key(getItemRequestKey) - .build(); - GetItemResponse response = dynamoDbClient.getItem(itemRequest); + GetItemResponse response = getItem(jobType); AttributeValue value = response.item().get(key); if (value != null) { return value.s(); @@ -175,9 +170,9 @@ public String getValue(String jobType, String key) { public Set findAllJobTypes() { ScanRequest scanRequest = ScanRequest.builder() .tableName(JOB_META_TABLE_NAME) - .attributesToGet("jobType").build(); + .attributesToGet(JOB_TYPE_KEY).build(); ScanResponse scanResponse = dynamoDbClient.scan(scanRequest); - Set jobTypes = scanResponse.items().stream().map(m -> m.get("jobType").s()).collect(Collectors.toSet()); + Set jobTypes = scanResponse.items().stream().map(m -> m.get(JOB_TYPE_KEY).s()).collect(Collectors.toSet()); return jobTypes; } @@ -191,11 +186,11 @@ private void createTable() { dynamoDbClient.createTable(CreateTableRequest.builder() .tableName(JOB_META_TABLE_NAME) .attributeDefinitions(AttributeDefinition.builder() - .attributeName("jobType") + .attributeName(JOB_TYPE_KEY) .attributeType(ScalarAttributeType.S) .build()) .keySchema(KeySchemaElement.builder() - .attributeName("jobType") + .attributeName(JOB_TYPE_KEY) .keyType(KeyType.HASH) .build()) .provisionedThroughput(ProvisionedThroughput.builder() @@ -210,4 +205,13 @@ private void deleteTable() { .tableName(JOB_META_TABLE_NAME).build(); dynamoDbClient.deleteTable(deleteTableRequest); } + + private GetItemResponse getItem(String jobType) { + ImmutableMap itemRequestKey = ImmutableMap.of(JOB_TYPE_KEY, toAttributeValue(jobType)); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName(JOB_META_TABLE_NAME) + .key(itemRequestKey) + .build(); + return dynamoDbClient.getItem(itemRequest); + } } From 7c4aa0aa7ebe50a2ee758a282c8681c2a403980c Mon Sep 17 00:00:00 2001 From: tmoeller Date: Wed, 13 Nov 2019 10:06:06 +0100 Subject: [PATCH 56/78] tm|task|implement untested method for DynamoJobRepo --- .../dynamo/DynamoJobRepository.java | 106 ++++++++++-------- .../dynamo/DynamoJobRepositoryTest.java | 38 +++++++ 2 files changed, 97 insertions(+), 47 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 34033e9ff..5f430b569 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -12,6 +12,7 @@ import java.util.*; import java.util.stream.Collectors; +import static de.otto.edison.jobs.repository.dynamo.JobStructure.*; import static java.util.Collections.emptyList; import static java.util.Comparator.comparingLong; import static java.util.stream.Collectors.*; @@ -37,19 +38,17 @@ public DynamoJobRepository(DynamoDbClient dynamoDbClient, int pageSize) { @Override public Optional findOne(String jobId) { Map keyMap = new HashMap<>(); - keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); - + keyMap.put(ID.key(), toStringAttributeValue(jobId)); GetItemRequest jobInfoRequest = GetItemRequest.builder() .tableName(JOBS_TABLE_NAME) .key(keyMap) .build(); + final GetItemResponse jobInfoResponse = dynamoDbClient.getItem(jobInfoRequest); if (jobInfoResponse.item().isEmpty()) { return Optional.empty(); } - final JobInfo jobInfo = decode(jobInfoResponse.item()); - return Optional.of(jobInfo); - + return Optional.of(decode(jobInfoResponse.item())); } @Override @@ -95,7 +94,7 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .expressionAttributeValues(expressionAttributeValues) - .filterExpression(JobStructure.LAST_UPDATED_EPOCH.key() + " < :val and attribute_not_exists(" + JobStructure.STOPPED.key() + ")") + .filterExpression(LAST_UPDATED_EPOCH.key() + " < :val and attribute_not_exists(" + STOPPED.key() + ")") .build(); final ScanResponse response = dynamoDbClient.scan(query); @@ -106,29 +105,45 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { return jobs; } - @Override - public List findAll() { + + private List findAll(final boolean withMessages) { Map lastKeyEvaluated = null; List jobs = new ArrayList<>(); do { - ScanRequest findAll = ScanRequest.builder() + final ScanRequest.Builder findAllRequestBuilder = ScanRequest.builder() .tableName(JOBS_TABLE_NAME) .limit(pageSize) - .exclusiveStartKey(lastKeyEvaluated) - .build(); + .exclusiveStartKey(lastKeyEvaluated); + + if (!withMessages) { + String projectionExpressionBuilder = ID.key() + + ", " + STARTED.key() + + ", " + STOPPED.key() + + ", " + JOB_TYPE.key() + + ", #" + STATUS.key() + + ", " + HOSTNAME.key() + + ", " + LAST_UPDATED.key() + + ", " + LAST_UPDATED_EPOCH.key(); + findAllRequestBuilder.projectionExpression(projectionExpressionBuilder) + .expressionAttributeNames(ImmutableMap.of("#" + STATUS.key(), STATUS.key())); + } - final ScanResponse scan = dynamoDbClient.scan(findAll); + final ScanResponse scan = dynamoDbClient.scan(findAllRequestBuilder.build()); lastKeyEvaluated = scan.lastEvaluatedKey(); List newJobsFromThisPage = scan.items().stream().map(this::decode).collect(Collectors.toList()); jobs.addAll(newJobsFromThisPage); } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); - return jobs; } + @Override + public List findAll() { + return findAll(true); + } + @Override public List findAllJobInfoWithoutMessages() { - return null; + return findAll(false); } @Override @@ -144,7 +159,7 @@ public List findByType(String jobType) { .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .expressionAttributeValues(expressionAttributeValues) - .filterExpression(JobStructure.JOB_TYPE.key() + " = :jobType") + .filterExpression(JOB_TYPE.key() + " = :jobType") .build(); final ScanResponse response = dynamoDbClient.scan(query); @@ -166,59 +181,56 @@ public JobInfo createOrUpdate(JobInfo job) { dynamoDbClient.putItem(putItemRequest); return job; - } private Map encode(JobInfo jobInfo) { Map jobAsItem = new HashMap<>(); - jobAsItem.put(JobStructure.ID.key(), toStringAttributeValue(jobInfo.getJobId())); - jobAsItem.put(JobStructure.HOSTNAME.key(), toStringAttributeValue(jobInfo.getHostname())); - jobAsItem.put(JobStructure.JOB_TYPE.key(), toStringAttributeValue(jobInfo.getJobType())); - jobAsItem.put(JobStructure.STARTED.key(), toStringAttributeValue(jobInfo.getStarted())); - jobAsItem.put(JobStructure.STATUS.key(), toStringAttributeValue(jobInfo.getStatus().name())); - jobInfo.getStopped().ifPresent(offsetDateTime -> jobAsItem.put(JobStructure.STOPPED.key(), toStringAttributeValue(offsetDateTime))); + jobAsItem.put(ID.key(), toStringAttributeValue(jobInfo.getJobId())); + jobAsItem.put(HOSTNAME.key(), toStringAttributeValue(jobInfo.getHostname())); + jobAsItem.put(JOB_TYPE.key(), toStringAttributeValue(jobInfo.getJobType())); + jobAsItem.put(STARTED.key(), toStringAttributeValue(jobInfo.getStarted())); + jobAsItem.put(STATUS.key(), toStringAttributeValue(jobInfo.getStatus().name())); + jobInfo.getStopped().ifPresent(offsetDateTime -> jobAsItem.put(STOPPED.key(), toStringAttributeValue(offsetDateTime))); if (null != jobInfo.getLastUpdated()) { - jobAsItem.put(JobStructure.LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); - jobAsItem.put(JobStructure.LAST_UPDATED_EPOCH.key(), toNumberAttributeValue(jobInfo.getLastUpdated().toInstant().toEpochMilli())); + jobAsItem.put(LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); + jobAsItem.put(LAST_UPDATED_EPOCH.key(), toNumberAttributeValue(jobInfo.getLastUpdated().toInstant().toEpochMilli())); } - jobAsItem.put(JobStructure.MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); - - + jobAsItem.put(MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); return jobAsItem; } private JobInfo decode(Map item) { final JobInfo.Builder jobInfo = JobInfo.builder() - .setJobId(item.get(JobStructure.ID.key()).s()) - .setHostname(item.get(JobStructure.HOSTNAME.key()).s()) - .setJobType(item.get(JobStructure.JOB_TYPE.key()).s()) - .setStarted(OffsetDateTime.parse(item.get(JobStructure.STARTED.key()).s())) - .setStatus(JobInfo.JobStatus.valueOf(item.get(JobStructure.STATUS.key()).s())) + .setJobId(item.get(ID.key()).s()) + .setHostname(item.get(HOSTNAME.key()).s()) + .setJobType(item.get(JOB_TYPE.key()).s()) + .setStarted(OffsetDateTime.parse(item.get(STARTED.key()).s())) + .setStatus(JobInfo.JobStatus.valueOf(item.get(STATUS.key()).s())) .setMessages(itemToJobMessages(item)); - if (item.containsKey(JobStructure.STOPPED.key())) { - jobInfo.setStopped(OffsetDateTime.parse(item.get(JobStructure.STOPPED.key()).s())); + if (item.containsKey(STOPPED.key())) { + jobInfo.setStopped(OffsetDateTime.parse(item.get(STOPPED.key()).s())); } - if (item.containsKey(JobStructure.LAST_UPDATED.key())) { - jobInfo.setLastUpdated(OffsetDateTime.parse(item.get(JobStructure.LAST_UPDATED.key()).s())); + if (item.containsKey(LAST_UPDATED.key())) { + jobInfo.setLastUpdated(OffsetDateTime.parse(item.get(LAST_UPDATED.key()).s())); } return jobInfo.build(); } private List itemToJobMessages(Map item) { - if (!item.containsKey(JobStructure.MESSAGES.key())) { + if (!item.containsKey(MESSAGES.key())) { return emptyList(); } - final AttributeValue attributeValue = item.get(JobStructure.MESSAGES.key()); + final AttributeValue attributeValue = item.get(MESSAGES.key()); return attributeValue.l().stream().map(this::attributeValueToMessage).collect(Collectors.toList()); } private JobMessage attributeValueToMessage(AttributeValue attributeValue) { final Map messageMap = attributeValue.m(); - final Level level = Level.ofKey(messageMap.get(JobStructure.MSG_LEVEL.key()).s()); - final String text = messageMap.get(JobStructure.MSG_TEXT.key()).s(); - final OffsetDateTime timestamp = OffsetDateTime.parse(messageMap.get(JobStructure.MSG_TS.key()).s()); + final Level level = Level.ofKey(messageMap.get(MSG_LEVEL.key()).s()); + final String text = messageMap.get(MSG_TEXT.key()).s(); + final OffsetDateTime timestamp = OffsetDateTime.parse(messageMap.get(MSG_TS.key()).s()); return JobMessage.jobMessage(level, text, timestamp); } @@ -227,7 +239,7 @@ public void removeIfStopped(String jobId) { findOne(jobId).ifPresent(jobInfo -> { if (jobInfo.isStopped()) { Map keyMap = new HashMap<>(); - keyMap.put(JobStructure.ID.key(), toStringAttributeValue(jobId)); + keyMap.put(ID.key(), toStringAttributeValue(jobId)); DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() .tableName(JOBS_TABLE_NAME) .key(keyMap) @@ -305,11 +317,11 @@ private void createTable() { dynamoDbClient.createTable(CreateTableRequest.builder() .tableName(JOBS_TABLE_NAME) .attributeDefinitions(AttributeDefinition.builder() - .attributeName(JobStructure.ID.key()) + .attributeName(ID.key()) .attributeType(ScalarAttributeType.S) .build()) .keySchema(KeySchemaElement.builder() - .attributeName(JobStructure.ID.key()) + .attributeName(ID.key()) .keyType(KeyType.HASH) .build()) .provisionedThroughput(ProvisionedThroughput.builder() @@ -333,9 +345,9 @@ private AttributeValue toNumberAttributeValue(long value) { private AttributeValue toMapAttributeValue(JobMessage jobMessage) { Map message = new HashMap<>(); - message.put(JobStructure.MSG_LEVEL.key(), toStringAttributeValue(jobMessage.getLevel().getKey())); - message.put(JobStructure.MSG_TEXT.key(), toStringAttributeValue(jobMessage.getMessage())); - message.put(JobStructure.MSG_TS.key(), toStringAttributeValue(jobMessage.getTimestamp())); + message.put(MSG_LEVEL.key(), toStringAttributeValue(jobMessage.getLevel().getKey())); + message.put(MSG_TEXT.key(), toStringAttributeValue(jobMessage.getMessage())); + message.put(MSG_TS.key(), toStringAttributeValue(jobMessage.getTimestamp())); return AttributeValue.builder() .m(message) .build(); diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 0ecbe5114..a0d1e3fbf 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -401,6 +401,44 @@ void shouldClearJobInfos() { assertThat(testee.findAll(), is(emptyList())); } + @Test + public void shouldStoreAndRetrieveAllJobInfoWithoutMessages() { + // given + JobInfo job1 = builder() + .setJobId("someJobId1") + .setJobType("someJobType1") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .setLastUpdated(OffsetDateTime.now()) + .setClock(clock) + .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage", OffsetDateTime.now())) + .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage", OffsetDateTime.now().plusSeconds(5L))) + .build(); + JobInfo job2 = builder() + .setJobId("someJobId2") + .setJobType("someJobType2") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .setLastUpdated(OffsetDateTime.now()) + .setClock(clock) + .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage", OffsetDateTime.now())) + .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage", OffsetDateTime.now().plusSeconds(5L))) + .build(); + testee.createOrUpdate(job1); + testee.createOrUpdate(job2); + + // when + final List jobInfos = testee.findAllJobInfoWithoutMessages(); + // then + assertThat(jobInfos, hasSize(2)); + assertThat(jobInfos.get(0), is(job2.copy().setMessages(emptyList()).build())); + assertThat(jobInfos.get(1), is(job1.copy().setMessages(emptyList()).build())); + } + private JobInfo jobInfo(final String jobId, final String type) { return JobInfo.newJobInfo( jobId, From 5ceb4c3532ab5ac9f1ea8f15d87b5c7616483463 Mon Sep 17 00:00:00 2001 From: tmoeller Date: Wed, 13 Nov 2019 10:13:51 +0100 Subject: [PATCH 57/78] tm|task|remove unused dynamo projection field --- .../jobs/repository/dynamo/DynamoJobRepository.java | 3 +-- .../repository/dynamo/DynamoJobRepositoryTest.java | 13 +++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 5f430b569..5f832f97a 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -122,8 +122,7 @@ private List findAll(final boolean withMessages) { ", " + JOB_TYPE.key() + ", #" + STATUS.key() + ", " + HOSTNAME.key() + - ", " + LAST_UPDATED.key() + - ", " + LAST_UPDATED_EPOCH.key(); + ", " + LAST_UPDATED.key(); findAllRequestBuilder.projectionExpression(projectionExpressionBuilder) .expressionAttributeNames(ImmutableMap.of("#" + STATUS.key(), STATUS.key())); } diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index a0d1e3fbf..b0d930c65 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -412,21 +412,18 @@ public void shouldStoreAndRetrieveAllJobInfoWithoutMessages() { .setHostname("localhost") .setStatus(JobStatus.OK) .setLastUpdated(OffsetDateTime.now()) - .setClock(clock) - .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage", OffsetDateTime.now())) - .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage", OffsetDateTime.now().plusSeconds(5L))) + .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage1", OffsetDateTime.now())) + .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage1", OffsetDateTime.now().plusSeconds(5L))) .build(); JobInfo job2 = builder() .setJobId("someJobId2") .setJobType("someJobType2") .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) - .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) .setHostname("localhost") - .setStatus(JobStatus.OK) + .setStatus(JobStatus.DEAD) .setLastUpdated(OffsetDateTime.now()) - .setClock(clock) - .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage", OffsetDateTime.now())) - .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage", OffsetDateTime.now().plusSeconds(5L))) + .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage2", OffsetDateTime.now())) + .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage2", OffsetDateTime.now().plusSeconds(5L))) .build(); testee.createOrUpdate(job1); testee.createOrUpdate(job2); From b5ee7c5212244f835663bcd8b54892b08ddadd3c Mon Sep 17 00:00:00 2001 From: tmoeller Date: Wed, 13 Nov 2019 15:16:33 +0100 Subject: [PATCH 58/78] bs/tm|task|add transactional functionality to dynamoJobRepo --- .../dynamo/DynamoJobRepository.java | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 5f832f97a..68e2568cb 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -20,6 +20,7 @@ public class DynamoJobRepository implements JobRepository { private static final String JOBS_TABLE_NAME = "FT6_DynamoDB_Jobs"; + private static final String ETAG_KEY = "etag"; private final DynamoDbClient dynamoDbClient; private final int pageSize; @@ -37,6 +38,10 @@ public DynamoJobRepository(DynamoDbClient dynamoDbClient, int pageSize) { @Override public Optional findOne(String jobId) { + return findOneItem(jobId).map(this::decode); + } + + private Optional> findOneItem(final String jobId) { Map keyMap = new HashMap<>(); keyMap.put(ID.key(), toStringAttributeValue(jobId)); GetItemRequest jobInfoRequest = GetItemRequest.builder() @@ -48,7 +53,7 @@ public Optional findOne(String jobId) { if (jobInfoResponse.item().isEmpty()) { return Optional.empty(); } - return Optional.of(decode(jobInfoResponse.item())); + return Optional.of(jobInfoResponse.item()); } @Override @@ -170,15 +175,28 @@ public List findByType(String jobType) { } @Override - public JobInfo createOrUpdate(JobInfo job) { - + public JobInfo createOrUpdate(final JobInfo job) { Map jobAsItem = encode(job); PutItemRequest putItemRequest = PutItemRequest.builder() .tableName(JOBS_TABLE_NAME) .item(jobAsItem) .build(); dynamoDbClient.putItem(putItemRequest); + return job; + } + public JobInfo createOrUpdate(final JobInfo job, final AttributeValue etag) { + Map jobAsItem = encode(job); + final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() + .tableName(JOBS_TABLE_NAME) + .item(jobAsItem); + if (etag != null) { + Map valueMap = new HashMap<>(); + valueMap.put(":val", AttributeValue.builder().s(etag.s()).build()); + putItemRequestBuilder.expressionAttributeValues(valueMap); + putItemRequestBuilder.conditionExpression("contains(etag, :val)"); + } + dynamoDbClient.putItem(putItemRequestBuilder.build()); return job; } @@ -257,27 +275,34 @@ public JobInfo.JobStatus findStatus(String jobId) { @Override public void appendMessage(String jobId, JobMessage jobMessage) { - JobInfo jobInfo = findOne(jobId).orElseThrow(RuntimeException::new); - createOrUpdate(jobInfo.copy() + final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); + JobInfo jobInfo = decode(item); + createOrUpdate( + jobInfo.copy() .addMessage(jobMessage) .setLastUpdated(jobMessage.getTimestamp()) - .build()); + .build(), + item.get(ETAG_KEY)); } @Override public void setJobStatus(String jobId, JobInfo.JobStatus jobStatus) { - JobInfo jobInfo = findOne(jobId).orElseThrow(RuntimeException::new); + final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); + JobInfo jobInfo = decode(item); createOrUpdate(jobInfo.copy() .setStatus(jobStatus) - .build()); + .build(), + item.get(ETAG_KEY)); } @Override public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { - JobInfo jobInfo = findOne(jobId).orElseThrow(RuntimeException::new); + final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); + JobInfo jobInfo = decode(item); createOrUpdate(jobInfo.copy() .setLastUpdated(lastUpdate) - .build()); + .build(), + item.get(ETAG_KEY)); } @Override From c03e092a76fef1739d513775b4f4bcb3c4515696 Mon Sep 17 00:00:00 2001 From: tmoeller Date: Mon, 18 Nov 2019 14:30:18 +0100 Subject: [PATCH 59/78] rs/tm|task|make dynamodb configurable --- .../DynamoJobsConfiguration.java | 43 +++++++++++++++++++ .../dynamo/DynamoJobMetaRepository.java | 27 +++++------- .../dynamo/DynamoJobRepository.java | 32 ++++++-------- .../dynamo/DynamoJobRepositoryTest.java | 10 ++--- .../mongo/JobMetaRepositoryTest.java | 4 +- 5 files changed, 74 insertions(+), 42 deletions(-) create mode 100644 edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java b/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java new file mode 100644 index 000000000..fe715b477 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java @@ -0,0 +1,43 @@ +package de.otto.edison.jobs.configuration; + +import de.otto.edison.jobs.repository.JobMetaRepository; +import de.otto.edison.jobs.repository.JobRepository; +import de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository; +import de.otto.edison.jobs.repository.dynamo.DynamoJobRepository; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import static org.slf4j.LoggerFactory.getLogger; + +@Configuration +@ConditionalOnProperty(prefix = "edison.jobs", name = "dynamo.enabled", havingValue = "true", matchIfMissing = true) +@ConditionalOnBean(type = "software.amazon.awssdk.services.dynamodb.DynamoDbClient") +public class DynamoJobsConfiguration { + + private static final Logger LOG = getLogger(DynamoJobsConfiguration.class); + + @Bean + public JobRepository jobRepository(final DynamoDbClient dynamoDbClient, + final @Value("${edison.jobs.dynamo.jobinfo.tableName}") String tableName, + final @Value("${edison.jobs.dynamo.jobinfo.pageSize}") int pageSize) { + LOG.info("==============================="); + LOG.info("Using DynamoJobRepository with tableName {} and pageSize {}.",tableName, pageSize); + LOG.info("==============================="); + return new DynamoJobRepository(dynamoDbClient, tableName, pageSize); + } + + @Bean + public JobMetaRepository jobMetaRepository(final DynamoDbClient dynamoDbClient, + final @Value("${edison.jobs.dynamo.jobmeta.tableName}") String tableName) { + LOG.info("==============================="); + LOG.info("Using DynamoJobMetaRepository with tableName {}.", tableName); + LOG.info("==============================="); + return new DynamoJobMetaRepository(dynamoDbClient, tableName); + } + +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index c1e56537b..d0f871fe8 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -19,16 +19,17 @@ public class DynamoJobMetaRepository implements JobMetaRepository { private static final String KEY_PREFIX = "_e_"; private static final String KEY_DISABLED = KEY_PREFIX + "disabled"; private static final String KEY_RUNNING = KEY_PREFIX + "running"; - private static final String JOB_META_TABLE_NAME = "FT6_DynamoDB_JobMeta"; private static final String JOB_TYPE_KEY = "jobType"; private static final String ETAG_KEY = "etag"; private final DynamoDbClient dynamoDbClient; + private final String tableName; - public DynamoJobMetaRepository(DynamoDbClient dynamoDbClient) { + public DynamoJobMetaRepository(final DynamoDbClient dynamoDbClient, final String tableName) { this.dynamoDbClient = dynamoDbClient; + this.tableName = tableName; try { dynamoDbClient.describeTable(DescribeTableRequest.builder() - .tableName(JOB_META_TABLE_NAME) + .tableName(tableName) .build()); } catch (ResourceNotFoundException e) { createTable(); @@ -40,7 +41,6 @@ public JobMeta getJobMeta(String jobType) { GetItemResponse response = getItem(jobType); Map responseItem = response.item(); if (!responseItem.isEmpty()) { - final boolean isRunning = responseItem.containsKey(KEY_RUNNING); final boolean isDisabled = responseItem.containsKey(KEY_DISABLED); final AttributeValue attributeValueComment = responseItem.get(KEY_DISABLED); @@ -48,7 +48,6 @@ public JobMeta getJobMeta(String jobType) { if (attributeValueComment != null) { comment = attributeValueComment.s(); } - Map metaMap = responseItem.entrySet().stream() .filter(e -> !e.getKey().startsWith(KEY_PREFIX)) .filter(e -> !e.getKey().equals(JOB_TYPE_KEY)) @@ -88,7 +87,6 @@ public void clearRunningJob(String jobType) { @Override public void disable(String jobType, String comment) { setValue(jobType, KEY_DISABLED, comment != null ? comment : ""); - } @Override @@ -104,7 +102,6 @@ public String setValue(String jobType, String key, String value) { private String putValue(String jobType, String key, String value) { GetItemResponse getItemResponse = getItem(jobType); - Map newEntry = new HashMap<>(getItemResponse.item()); if (value == null) { newEntry.remove(key); @@ -112,15 +109,11 @@ private String putValue(String jobType, String key, String value) { newEntry.put(key, toAttributeValue(value)); } newEntry.put(ETAG_KEY, AttributeValue.builder().s(UUID.randomUUID().toString()).build()); - final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) + .tableName(tableName) .item(newEntry); - addEtagCondition(putItemRequestBuilder, getItemResponse); - dynamoDbClient.putItem(putItemRequestBuilder.build()); - AttributeValue existingValueForKey = getItemResponse.item().get(key); if (existingValueForKey == null) { return null; @@ -148,7 +141,7 @@ private void putIfAbsent(String jobType) { Map item = new HashMap<>(); item.put(JOB_TYPE_KEY, toAttributeValue(jobType)); PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) + .tableName(tableName) .item(item) .build(); dynamoDbClient.putItem(putItemRequest); @@ -169,7 +162,7 @@ public String getValue(String jobType, String key) { @Override public Set findAllJobTypes() { ScanRequest scanRequest = ScanRequest.builder() - .tableName(JOB_META_TABLE_NAME) + .tableName(tableName) .attributesToGet(JOB_TYPE_KEY).build(); ScanResponse scanResponse = dynamoDbClient.scan(scanRequest); Set jobTypes = scanResponse.items().stream().map(m -> m.get(JOB_TYPE_KEY).s()).collect(Collectors.toSet()); @@ -184,7 +177,7 @@ public void deleteAll() { private void createTable() { dynamoDbClient.createTable(CreateTableRequest.builder() - .tableName(JOB_META_TABLE_NAME) + .tableName(tableName) .attributeDefinitions(AttributeDefinition.builder() .attributeName(JOB_TYPE_KEY) .attributeType(ScalarAttributeType.S) @@ -202,14 +195,14 @@ private void createTable() { private void deleteTable() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(JOB_META_TABLE_NAME).build(); + .tableName(tableName).build(); dynamoDbClient.deleteTable(deleteTableRequest); } private GetItemResponse getItem(String jobType) { ImmutableMap itemRequestKey = ImmutableMap.of(JOB_TYPE_KEY, toAttributeValue(jobType)); GetItemRequest itemRequest = GetItemRequest.builder() - .tableName(JOB_META_TABLE_NAME) + .tableName(tableName) .key(itemRequestKey) .build(); return dynamoDbClient.getItem(itemRequest); diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 68e2568cb..9cad8c996 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -19,17 +19,18 @@ public class DynamoJobRepository implements JobRepository { - private static final String JOBS_TABLE_NAME = "FT6_DynamoDB_Jobs"; private static final String ETAG_KEY = "etag"; private final DynamoDbClient dynamoDbClient; + private final String tableName; private final int pageSize; - public DynamoJobRepository(DynamoDbClient dynamoDbClient, int pageSize) { + public DynamoJobRepository(DynamoDbClient dynamoDbClient, final String tableName, int pageSize) { this.dynamoDbClient = dynamoDbClient; + this.tableName = tableName; this.pageSize = pageSize; try { dynamoDbClient.describeTable(DescribeTableRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .build()); } catch (ResourceNotFoundException e) { createTable(); @@ -45,10 +46,9 @@ private Optional> findOneItem(final String jobId) { Map keyMap = new HashMap<>(); keyMap.put(ID.key(), toStringAttributeValue(jobId)); GetItemRequest jobInfoRequest = GetItemRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .key(keyMap) .build(); - final GetItemResponse jobInfoResponse = dynamoDbClient.getItem(jobInfoRequest); if (jobInfoResponse.item().isEmpty()) { return Optional.empty(); @@ -95,7 +95,7 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { ); do { final ScanRequest query = ScanRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .expressionAttributeValues(expressionAttributeValues) @@ -116,7 +116,7 @@ private List findAll(final boolean withMessages) { List jobs = new ArrayList<>(); do { final ScanRequest.Builder findAllRequestBuilder = ScanRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated); @@ -131,7 +131,6 @@ private List findAll(final boolean withMessages) { findAllRequestBuilder.projectionExpression(projectionExpressionBuilder) .expressionAttributeNames(ImmutableMap.of("#" + STATUS.key(), STATUS.key())); } - final ScanResponse scan = dynamoDbClient.scan(findAllRequestBuilder.build()); lastKeyEvaluated = scan.lastEvaluatedKey(); List newJobsFromThisPage = scan.items().stream().map(this::decode).collect(Collectors.toList()); @@ -159,7 +158,7 @@ public List findByType(String jobType) { ); do { final ScanRequest query = ScanRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) .expressionAttributeValues(expressionAttributeValues) @@ -178,7 +177,7 @@ public List findByType(String jobType) { public JobInfo createOrUpdate(final JobInfo job) { Map jobAsItem = encode(job); PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .item(jobAsItem) .build(); dynamoDbClient.putItem(putItemRequest); @@ -188,7 +187,7 @@ public JobInfo createOrUpdate(final JobInfo job) { public JobInfo createOrUpdate(final JobInfo job, final AttributeValue etag) { Map jobAsItem = encode(job); final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .item(jobAsItem); if (etag != null) { Map valueMap = new HashMap<>(); @@ -227,7 +226,6 @@ private JobInfo decode(Map item) { if (item.containsKey(STOPPED.key())) { jobInfo.setStopped(OffsetDateTime.parse(item.get(STOPPED.key()).s())); } - if (item.containsKey(LAST_UPDATED.key())) { jobInfo.setLastUpdated(OffsetDateTime.parse(item.get(LAST_UPDATED.key()).s())); } @@ -238,7 +236,6 @@ private List itemToJobMessages(Map item) { if (!item.containsKey(MESSAGES.key())) { return emptyList(); } - final AttributeValue attributeValue = item.get(MESSAGES.key()); return attributeValue.l().stream().map(this::attributeValueToMessage).collect(Collectors.toList()); } @@ -258,7 +255,7 @@ public void removeIfStopped(String jobId) { Map keyMap = new HashMap<>(); keyMap.put(ID.key(), toStringAttributeValue(jobId)); DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .key(keyMap) .build(); dynamoDbClient.deleteItem(deleteJobRequest); @@ -311,7 +308,7 @@ public long size() { long count = 0; do { ScanRequest counterQuery = ScanRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .select(Select.COUNT) .limit(pageSize) .exclusiveStartKey(lastKeyEvaluated) @@ -321,7 +318,6 @@ public long size() { lastKeyEvaluated = countResponse.lastEvaluatedKey(); count = count + countResponse.count(); } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); - return count; } @@ -333,13 +329,13 @@ public void deleteAll() { private void deleteTable() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(JOBS_TABLE_NAME).build(); + .tableName(tableName).build(); dynamoDbClient.deleteTable(deleteTableRequest); } private void createTable() { dynamoDbClient.createTable(CreateTableRequest.builder() - .tableName(JOBS_TABLE_NAME) + .tableName(tableName) .attributeDefinitions(AttributeDefinition.builder() .attributeName(ID.key()) .attributeType(ScalarAttributeType.S) diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index b0d930c65..46dd09efe 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -64,7 +64,7 @@ void tearDown() { @BeforeEach void setUp() { - testee = new DynamoJobRepository(getDynamoDbClient(), 10); + testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); } private static DynamoDbClient getDynamoDbClient() { @@ -98,7 +98,7 @@ void shouldCreateOrUpdateJob() { @Test void shouldFindJobInfoByUri() { // given - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), 10); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); // when JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); @@ -110,7 +110,7 @@ void shouldFindJobInfoByUri() { @Test void shouldReturnAbsentStatus() { - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), 10); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(),JOBS_TABLE_NAME, 10); assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); } @@ -168,7 +168,7 @@ void shouldFindAll() { @Test void shouldFindAllWithPaging() { // given - testee = new DynamoJobRepository(getDynamoDbClient(), 2); + testee = new DynamoJobRepository(getDynamoDbClient(),JOBS_TABLE_NAME, 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest1", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -194,7 +194,7 @@ void shouldFindAllinSizeOperation() { @Test void shouldFindAllinSizeOperationWithPageing() { // given - testee = new DynamoJobRepository(getDynamoDbClient(), 2); + testee = new DynamoJobRepository(getDynamoDbClient(),JOBS_TABLE_NAME, 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youn44444556gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index 8c3d543b3..484a1ccb8 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -48,7 +48,7 @@ public static void teardownMongo() { @BeforeAll public static void initDbs() throws IOException { EmbeddedMongoHelper.startMongoDB(); - dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient()); + dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); } @Container @@ -57,7 +57,7 @@ public static void initDbs() throws IOException { @BeforeEach void setUp() { - dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient()); + dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); } @AfterEach From fe4f080082f965c0ff0409628157bea19311b94d Mon Sep 17 00:00:00 2001 From: tmoeller Date: Mon, 18 Nov 2019 14:46:42 +0100 Subject: [PATCH 60/78] rs/tm|task|disable dynamo job repos by default, add changelog --- CHANGELOG.md | 9 +++++++++ .../jobs/configuration/DynamoJobsConfiguration.java | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fad9e4ce..f33ac369f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Release Notes +## 2.2.2 +* **[edison-jobs]**: add DynamoDb Support for Edison-Jobs + * Properties for enabling DynamoDb: + * **edison.jobs.dynamo.enabled**: Enable DynamoDb (disabled by default) + * **edison.jobs.mongo.enabled**: MongoDb needs to be disabled + * **edison.jobs.dynamo.jobinfo.tableName**: Name for JobInfo table (gets created if non-existent) + * **edison.jobs.dynamo.jobinfo.pageSize**: PageSize for scan-requests against JobInfo table + * **edison.jobs.dynamo.jobmeta.tableName**: Name for JobMeta table (gets created if non-existent) + ## 2.2.1 * **[general]**: upgrade aws sdk * **[edison-core]**: Fix basic auth credentials retrieval on wrong format diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java b/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java index fe715b477..8be454000 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java @@ -15,7 +15,7 @@ import static org.slf4j.LoggerFactory.getLogger; @Configuration -@ConditionalOnProperty(prefix = "edison.jobs", name = "dynamo.enabled", havingValue = "true", matchIfMissing = true) +@ConditionalOnProperty(prefix = "edison.jobs", name = "dynamo.enabled", havingValue = "true") @ConditionalOnBean(type = "software.amazon.awssdk.services.dynamodb.DynamoDbClient") public class DynamoJobsConfiguration { From 67dfdaf460be37a5ca0f422a71faf3605e4113d3 Mon Sep 17 00:00:00 2001 From: bs/sm Date: Wed, 8 Jan 2020 11:26:04 +0100 Subject: [PATCH 61/78] =?UTF-8?q?task=20=F0=9F=93=9D|bs/sm|add=20tests=20f?= =?UTF-8?q?or=20DynamoJobMetaRepo=20and=20handle=20null=20values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bennet Schulz Co-authored-by: Simon Monecke --- .../dynamo/DynamoJobMetaRepository.java | 21 +- .../dynamo/DynamoJobMetaRepositoryTest.java | 275 ++++++++++++++++++ 2 files changed, 283 insertions(+), 13 deletions(-) create mode 100644 edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index d0f871fe8..067897023 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -17,9 +17,9 @@ public class DynamoJobMetaRepository implements JobMetaRepository { private static final String KEY_PREFIX = "_e_"; - private static final String KEY_DISABLED = KEY_PREFIX + "disabled"; + static final String KEY_DISABLED = KEY_PREFIX + "disabled"; private static final String KEY_RUNNING = KEY_PREFIX + "running"; - private static final String JOB_TYPE_KEY = "jobType"; + static final String JOB_TYPE_KEY = "jobType"; private static final String ETAG_KEY = "etag"; private final DynamoDbClient dynamoDbClient; private final String tableName; @@ -86,7 +86,7 @@ public void clearRunningJob(String jobType) { @Override public void disable(String jobType, String comment) { - setValue(jobType, KEY_DISABLED, comment != null ? comment : ""); + setValue(jobType, KEY_DISABLED, comment); } @Override @@ -103,22 +103,17 @@ public String setValue(String jobType, String key, String value) { private String putValue(String jobType, String key, String value) { GetItemResponse getItemResponse = getItem(jobType); Map newEntry = new HashMap<>(getItemResponse.item()); - if (value == null) { - newEntry.remove(key); - } else { - newEntry.put(key, toAttributeValue(value)); - } + newEntry.put(key, toAttributeValue(value)); newEntry.put(ETAG_KEY, AttributeValue.builder().s(UUID.randomUUID().toString()).build()); + final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() .tableName(tableName) .item(newEntry); addEtagCondition(putItemRequestBuilder, getItemResponse); dynamoDbClient.putItem(putItemRequestBuilder.build()); + AttributeValue existingValueForKey = getItemResponse.item().get(key); - if (existingValueForKey == null) { - return null; - } - return existingValueForKey.s(); + return existingValueForKey == null ? null : existingValueForKey.s(); } private void addEtagCondition(final PutItemRequest.Builder putItemRequestBuilder, final GetItemResponse getItemResponse) { @@ -132,7 +127,7 @@ private void addEtagCondition(final PutItemRequest.Builder putItemRequestBuilder } private AttributeValue toAttributeValue(String value) { - return AttributeValue.builder().s(value).build(); + return value == null || value.isEmpty() ? AttributeValue.builder().nul(true).build() : AttributeValue.builder().s(value).build(); } private void putIfAbsent(String jobType) { diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java new file mode 100644 index 000000000..a8a3d6c94 --- /dev/null +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java @@ -0,0 +1,275 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobMeta; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.net.URI; +import java.util.Collections; +import java.util.Set; + +import static de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository.JOB_TYPE_KEY; +import static de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository.KEY_DISABLED; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertThat; + +@Testcontainers +public class DynamoJobMetaRepositoryTest { + + @Container + private static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + .withExposedPorts(8000); + + private static final String TABLE_NAME = "jobMeta"; + + private static DynamoJobMetaRepository dynamoJobMetaRepository; + + @BeforeEach + public void before() { + createJobInfoTable(); + dynamoJobMetaRepository = new DynamoJobMetaRepository(getDynamoDbClient(), TABLE_NAME); + } + + private static DynamoDbClient getDynamoDbClient() { + String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + + dynamodb.getMappedPort(8000); + + return DynamoDbClient.builder() + .endpointOverride(URI.create(endpointUri)) + .region(Region.EU_CENTRAL_1) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create("acc", "sec"))).build(); + } + + @AfterEach + void tearDown() { + deleteJobInfoTable(); + } + + private void createJobInfoTable() { + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(TABLE_NAME) + .keySchema( + KeySchemaElement.builder().attributeName(JOB_TYPE_KEY).keyType(KeyType.HASH).build() + ) + .attributeDefinitions( + AttributeDefinition.builder().attributeName(JOB_TYPE_KEY).attributeType(ScalarAttributeType.S).build() + ) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build()) + .build()); + } + + private void deleteJobInfoTable() { + getDynamoDbClient().deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build()); + } + + @Test + void shouldSetRunningJob() { + //when + boolean notRunning = dynamoJobMetaRepository.setRunningJob("myJobType", "myJobId"); + + + //given + String jobId = dynamoJobMetaRepository.getRunningJob("myJobType"); + assertThat(jobId, is("myJobId")); + assertThat(notRunning, is(true)); + } + + @Test + public void shouldNotSetRunningWhenIsAlreadyRunning() { + //given + dynamoJobMetaRepository.setRunningJob("myJobType", "otherJobId"); + + //when + boolean notRunning = dynamoJobMetaRepository.setRunningJob("myJobType", "myJobId"); + + //given + String jobId = dynamoJobMetaRepository.getRunningJob("myJobType"); + assertThat(jobId, is("otherJobId")); + assertThat(notRunning, is(false)); + } + + @Test + void shouldSetAndGetValue() { + //given + + //when + dynamoJobMetaRepository.setValue("myJobType", "someKey", "someValue"); + + //given + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, is("someValue")); + } + + @Test + public void shouldReturnPreviousValueOnSetValue() { + //given + dynamoJobMetaRepository.setValue("myJobType", "someKey", "someOldValue"); + + //when + String previousValue = dynamoJobMetaRepository.setValue("myJobType", "someKey", "someNewValue"); + + //given + assertThat(previousValue, is("someOldValue")); + } + + @Test + public void shouldRemoveKeyWhenValueIsNull() { + //given + dynamoJobMetaRepository.setValue("myJobType", "someKey", "someOldValue"); + + //when + String previousValue = dynamoJobMetaRepository.setValue("myJobType", "someKey", null); + + //given + assertThat(previousValue, is("someOldValue")); + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, nullValue()); + } + + @Test + public void createValueShouldAddKeyIfNotExists() { + + //when + boolean valueCreated = dynamoJobMetaRepository.createValue("myJobType", "someKey", "someValue"); + + //then + assertThat(valueCreated, is(true)); + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, is("someValue")); + } + + @Test + public void createValueShouldNotAddKeyIfAlreadyExists() { + //given + dynamoJobMetaRepository.createValue("myJobType", "someKey", "someExistingValue"); + + //when + boolean valueCreated = dynamoJobMetaRepository.createValue("myJobType", "someKey", "someOtherValue"); + + //then + assertThat(valueCreated, is(false)); + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, is("someExistingValue")); + } + + @Test + public void shouldSetDisabledWithComment() { + //when + dynamoJobMetaRepository.disable("someJobType", "someComment"); + + //then + String value = dynamoJobMetaRepository.getValue("someJobType", KEY_DISABLED); + assertThat(value, is("someComment")); + } + + @Test + public void shouldSetDisabledWithoutComment() { + //when + dynamoJobMetaRepository = new DynamoJobMetaRepository(getDynamoDbClient(), TABLE_NAME); + dynamoJobMetaRepository.disable("someJobType", null); + + //then + String value = dynamoJobMetaRepository.getValue("someJobType", KEY_DISABLED); + assertThat(value, nullValue()); + } + + @Test + public void shouldSetEnabled() { + //given + dynamoJobMetaRepository.disable("someJobType", "disabled"); + + //when + dynamoJobMetaRepository.enable("someJobType"); + + //then + String value = dynamoJobMetaRepository.getValue("someJobType", KEY_DISABLED); + assertThat(value, nullValue()); + } + + @Test + public void shouldClearRunningJob() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId"); + + //when + dynamoJobMetaRepository.clearRunningJob("someJobType"); + + //then + String jobId = dynamoJobMetaRepository.getRunningJob("someJobType"); + assertThat(jobId, nullValue()); + } + + @Test + public void shouldFindAllJobTypes() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId1"); + dynamoJobMetaRepository.setRunningJob("someOtherJobType", "someJobId2"); + dynamoJobMetaRepository.enable("oneMoreJobType"); + + //when + Set allJobTypes = dynamoJobMetaRepository.findAllJobTypes(); + + //then + assertThat(allJobTypes, contains("someJobType", "someOtherJobType", "oneMoreJobType")); + } + + @Test + public void shouldReturnJobMeta() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId"); + dynamoJobMetaRepository.disable("someJobType", "because"); + dynamoJobMetaRepository.setValue("someJobType", "foo", "bar"); + + //when + JobMeta jobMeta = dynamoJobMetaRepository.getJobMeta("someJobType"); + + //then + assertThat(jobMeta.getJobType(), is("someJobType")); + assertThat(jobMeta.isDisabled(), is(true)); + assertThat(jobMeta.getDisabledComment(), is("because")); + assertThat(jobMeta.isRunning(), is(true)); + assertThat(jobMeta.getAll(), is(ImmutableMap.of("foo", "bar"))); + } + + @Test + public void shouldSetDisabledCommentToEmptyStringWhenEnabled() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId"); + dynamoJobMetaRepository.setValue("someJobType", "foo", "bar"); + + //when + JobMeta jobMeta = dynamoJobMetaRepository.getJobMeta("someJobType"); + + //then + assertThat(jobMeta.getJobType(), is("someJobType")); + assertThat(jobMeta.isDisabled(), is(false)); + assertThat(jobMeta.getDisabledComment(), is("")); + } + + @Test + public void shouldReturnEmptyJobMetaWhenJobTypeDoesNotExist() { + //when + JobMeta jobMeta = dynamoJobMetaRepository.getJobMeta("someJobType"); + + //then + assertThat(jobMeta.getJobType(), is("someJobType")); + assertThat(jobMeta.isDisabled(), is(false)); + assertThat(jobMeta.getDisabledComment(), is("")); + assertThat(jobMeta.isRunning(), is(false)); + assertThat(jobMeta.getAll(), is(Collections.emptyMap())); + } +} \ No newline at end of file From 856bd80db9dc1e7bc5623a27ff6b6461bd224283 Mon Sep 17 00:00:00 2001 From: MatthiasGeissendoerfer1997 Date: Thu, 9 Jan 2020 15:36:03 +0100 Subject: [PATCH 62/78] release version 2.2.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thimo Tollmien Co-authored-by: Matthias Geißendörfer --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f5b73b204..d8af743d4 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.2-SNAPSHOT' + version = '2.2.2' group = 'de.otto.edison' repositories { From 67fe27edb6a8cad63c087850420efda9a673a38d Mon Sep 17 00:00:00 2001 From: MatthiasGeissendoerfer1997 Date: Thu, 9 Jan 2020 16:05:16 +0100 Subject: [PATCH 63/78] bump next snapshot version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thimo Tollmien Co-authored-by: Matthias Geißendörfer --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d8af743d4..909e17def 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.2' + version = '2.2.3-SNAPSHOT' group = 'de.otto.edison' repositories { From 3057c46a0a1990e8279fa16598a319db6df0252a Mon Sep 17 00:00:00 2001 From: fw/tm Date: Wed, 15 Jan 2020 15:39:53 +0100 Subject: [PATCH 64/78] =?UTF-8?q?task=20=F0=9F=93=9D|fw/tm|implement=20cle?= =?UTF-8?q?ar=20table=20logic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tony Moeller --- .../dynamo/AbstractDynamoRepository.java | 36 +++++++++ .../dynamo/DynamoJobMetaRepository.java | 59 ++++---------- .../dynamo/DynamoJobRepository.java | 79 +++++++------------ .../dynamo/DynamoJobMetaRepositoryTest.java | 20 +++++ .../dynamo/DynamoJobRepositoryTest.java | 60 ++++++++++---- 5 files changed, 144 insertions(+), 110 deletions(-) create mode 100644 edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java new file mode 100644 index 000000000..6e3f5f8a7 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java @@ -0,0 +1,36 @@ +package de.otto.edison.jobs.repository.dynamo; + +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest; +import software.amazon.awssdk.services.dynamodb.model.WriteRequest; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +abstract class AbstractDynamoRepository { + + final DynamoDbClient dynamoDbClient; + final String tableName; + + AbstractDynamoRepository(DynamoDbClient dynamoDbClient, String tableName) { + this.dynamoDbClient = dynamoDbClient; + this.tableName = tableName; + } + + void deleteEntriesPerBatch(List deleteRequests) { + final int chunkSize = 25; + final AtomicInteger counter = new AtomicInteger(); + final Collection> deleteRequestsSplittedByChunkSize = deleteRequests.stream() + .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / chunkSize)) + .values(); + + deleteRequestsSplittedByChunkSize.forEach + (currentDeleteRequests -> dynamoDbClient.batchWriteItem( + BatchWriteItemRequest.builder().requestItems( + ImmutableMap.of(tableName, currentDeleteRequests)).build())); + + } +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index 067897023..ca7cac07c 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -6,34 +6,25 @@ import software.amazon.awssdk.services.dynamodb.model.*; import software.amazon.awssdk.utils.ImmutableMap; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toList; -public class DynamoJobMetaRepository implements JobMetaRepository { +public class DynamoJobMetaRepository extends AbstractDynamoRepository implements JobMetaRepository { private static final String KEY_PREFIX = "_e_"; static final String KEY_DISABLED = KEY_PREFIX + "disabled"; private static final String KEY_RUNNING = KEY_PREFIX + "running"; static final String JOB_TYPE_KEY = "jobType"; private static final String ETAG_KEY = "etag"; - private final DynamoDbClient dynamoDbClient; - private final String tableName; public DynamoJobMetaRepository(final DynamoDbClient dynamoDbClient, final String tableName) { - this.dynamoDbClient = dynamoDbClient; - this.tableName = tableName; - try { - dynamoDbClient.describeTable(DescribeTableRequest.builder() - .tableName(tableName) - .build()); - } catch (ResourceNotFoundException e) { - createTable(); - } + super(dynamoDbClient, tableName); + dynamoDbClient.describeTable(DescribeTableRequest.builder() + .tableName(tableName) + .build()); } @Override @@ -166,32 +157,16 @@ public Set findAllJobTypes() { @Override public void deleteAll() { - deleteTable(); - createTable(); - } - - private void createTable() { - dynamoDbClient.createTable(CreateTableRequest.builder() - .tableName(tableName) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName(JOB_TYPE_KEY) - .attributeType(ScalarAttributeType.S) - .build()) - .keySchema(KeySchemaElement.builder() - .attributeName(JOB_TYPE_KEY) - .keyType(KeyType.HASH) - .build()) - .provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(10L) - .writeCapacityUnits(10L) - .build()) - .build()); - } - - private void deleteTable() { - DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(tableName).build(); - dynamoDbClient.deleteTable(deleteTableRequest); + final List deleteRequests = findAllJobTypes().stream() + .map(jobId -> WriteRequest.builder() + .deleteRequest( + DeleteRequest.builder() + .key(ImmutableMap.of(JOB_TYPE_KEY, AttributeValue.builder().s(jobId).build())) + .build() + ).build() + ).collect(toList()); + + deleteEntriesPerBatch(deleteRequests); } private GetItemResponse getItem(String jobType) { diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 9cad8c996..517690bef 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -17,24 +17,17 @@ import static java.util.Comparator.comparingLong; import static java.util.stream.Collectors.*; -public class DynamoJobRepository implements JobRepository { +public class DynamoJobRepository extends AbstractDynamoRepository implements JobRepository { private static final String ETAG_KEY = "etag"; - private final DynamoDbClient dynamoDbClient; - private final String tableName; private final int pageSize; public DynamoJobRepository(DynamoDbClient dynamoDbClient, final String tableName, int pageSize) { - this.dynamoDbClient = dynamoDbClient; - this.tableName = tableName; + super(dynamoDbClient, tableName); this.pageSize = pageSize; - try { - dynamoDbClient.describeTable(DescribeTableRequest.builder() - .tableName(tableName) - .build()); - } catch (ResourceNotFoundException e) { - createTable(); - } + dynamoDbClient.describeTable(DescribeTableRequest.builder() + .tableName(tableName) + .build()); } @Override @@ -61,7 +54,7 @@ public List findLatest(int maxCount) { return findAll().stream() .sorted(Comparator.comparingLong(jobInfo -> jobInfo.getStarted().toInstant().toEpochMilli()).reversed()) .limit(maxCount) - .collect(Collectors.toList()); + .collect(toList()); } @Override @@ -83,7 +76,7 @@ public List findLatestBy(String type, int maxCount) { .sorted(Comparator.comparingLong(jobInfo -> jobInfo.getStarted().toInstant().toEpochMilli()).reversed()) .limit(maxCount) - .collect(Collectors.toList()); + .collect(toList()); } @Override @@ -104,13 +97,12 @@ public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { final ScanResponse response = dynamoDbClient.scan(query); lastKeyEvaluated = response.lastEvaluatedKey(); - List newJobsFromThisPage = response.items().stream().map(this::decode).collect(Collectors.toList()); + List newJobsFromThisPage = response.items().stream().map(this::decode).collect(toList()); jobs.addAll(newJobsFromThisPage); } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); return jobs; } - private List findAll(final boolean withMessages) { Map lastKeyEvaluated = null; List jobs = new ArrayList<>(); @@ -167,7 +159,7 @@ public List findByType(String jobType) { final ScanResponse response = dynamoDbClient.scan(query); lastKeyEvaluated = response.lastEvaluatedKey(); - List newJobsFromThisPage = response.items().stream().map(this::decode).collect(Collectors.toList()); + List newJobsFromThisPage = response.items().stream().map(this::decode).collect(toList()); jobs.addAll(newJobsFromThisPage); } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); return jobs; @@ -237,7 +229,7 @@ private List itemToJobMessages(Map item) { return emptyList(); } final AttributeValue attributeValue = item.get(MESSAGES.key()); - return attributeValue.l().stream().map(this::attributeValueToMessage).collect(Collectors.toList()); + return attributeValue.l().stream().map(this::attributeValueToMessage).collect(toList()); } private JobMessage attributeValueToMessage(AttributeValue attributeValue) { @@ -276,9 +268,9 @@ public void appendMessage(String jobId, JobMessage jobMessage) { JobInfo jobInfo = decode(item); createOrUpdate( jobInfo.copy() - .addMessage(jobMessage) - .setLastUpdated(jobMessage.getTimestamp()) - .build(), + .addMessage(jobMessage) + .setLastUpdated(jobMessage.getTimestamp()) + .build(), item.get(ETAG_KEY)); } @@ -287,8 +279,8 @@ public void setJobStatus(String jobId, JobInfo.JobStatus jobStatus) { final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); JobInfo jobInfo = decode(item); createOrUpdate(jobInfo.copy() - .setStatus(jobStatus) - .build(), + .setStatus(jobStatus) + .build(), item.get(ETAG_KEY)); } @@ -297,8 +289,8 @@ public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); JobInfo jobInfo = decode(item); createOrUpdate(jobInfo.copy() - .setLastUpdated(lastUpdate) - .build(), + .setLastUpdated(lastUpdate) + .build(), item.get(ETAG_KEY)); } @@ -323,32 +315,17 @@ public long size() { @Override public void deleteAll() { - deleteTable(); - createTable(); - } - - private void deleteTable() { - DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(tableName).build(); - dynamoDbClient.deleteTable(deleteTableRequest); - } + final List deleteRequests = findAll().stream() + .map(JobInfo::getJobId) + .map(jobId -> WriteRequest.builder() + .deleteRequest( + DeleteRequest.builder() + .key(ImmutableMap.of(ID.key(), AttributeValue.builder().s(jobId).build())) + .build() + ).build() + ).collect(toList()); - private void createTable() { - dynamoDbClient.createTable(CreateTableRequest.builder() - .tableName(tableName) - .attributeDefinitions(AttributeDefinition.builder() - .attributeName(ID.key()) - .attributeType(ScalarAttributeType.S) - .build()) - .keySchema(KeySchemaElement.builder() - .attributeName(ID.key()) - .keyType(KeyType.HASH) - .build()) - .provisionedThroughput(ProvisionedThroughput.builder() - .readCapacityUnits(10L) - .writeCapacityUnits(10L) - .build()) - .build()); + deleteEntriesPerBatch(deleteRequests); } private AttributeValue toStringAttributeValue(OffsetDateTime value) { @@ -374,7 +351,7 @@ private AttributeValue toMapAttributeValue(JobMessage jobMessage) { } private AttributeValue messagesToAttributeValueList(List jobeMessages) { - final List messageAttributes = jobeMessages.stream().map(this::toMapAttributeValue).collect(Collectors.toList()); + final List messageAttributes = jobeMessages.stream().map(this::toMapAttributeValue).collect(toList()); return toAttributeValueList(messageAttributes); } diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java index a8a3d6c94..b26b40cae 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java @@ -1,6 +1,7 @@ package de.otto.edison.jobs.repository.dynamo; import de.otto.edison.jobs.domain.JobMeta; +import org.hamcrest.MatcherAssert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -20,6 +21,7 @@ import static de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository.JOB_TYPE_KEY; import static de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository.KEY_DISABLED; +import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.*; import static org.junit.Assert.assertThat; @@ -272,4 +274,22 @@ public void shouldReturnEmptyJobMetaWhenJobTypeDoesNotExist() { assertThat(jobMeta.isRunning(), is(false)); assertThat(jobMeta.getAll(), is(Collections.emptyMap())); } + + @Test + void shouldDeleteAll() { + //given + // 25 is the max delete batch size + for (int i = 0; i < 26; i++) { + String jobType = "someJobType" + i; + String key = "someKey" + i; + String value = "someValue" + i; + dynamoJobMetaRepository.createValue(jobType, key, value); + } + + //when + dynamoJobMetaRepository.deleteAll(); + + //then + MatcherAssert.assertThat(dynamoJobMetaRepository.findAllJobTypes(), is(emptySet())); + } } \ No newline at end of file diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 46dd09efe..307bbc9ef 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -16,7 +16,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; +import software.amazon.awssdk.services.dynamodb.model.*; import java.net.URI; import java.time.Clock; @@ -32,6 +32,7 @@ import static de.otto.edison.jobs.domain.JobInfo.builder; import static de.otto.edison.jobs.domain.JobInfo.newJobInfo; import static de.otto.edison.jobs.domain.JobMessage.jobMessage; +import static de.otto.edison.jobs.repository.dynamo.JobStructure.ID; import static de.otto.edison.testsupport.matcher.OptionalMatchers.isAbsent; import static de.otto.edison.testsupport.matcher.OptionalMatchers.isPresent; import static java.time.Clock.fixed; @@ -43,6 +44,7 @@ import static java.util.UUID.randomUUID; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; +import static software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType.S; @Testcontainers class DynamoJobRepositoryTest { @@ -59,11 +61,12 @@ class DynamoJobRepositoryTest { void tearDown() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() .tableName(JOBS_TABLE_NAME).build(); - getDynamoDbClient().deleteTable(deleteTableRequest); + deleteJobInfoTable(); } @BeforeEach void setUp() { + createJobInfoTable(); testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); } @@ -98,7 +101,7 @@ void shouldCreateOrUpdateJob() { @Test void shouldFindJobInfoByUri() { // given - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); // when JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); @@ -110,7 +113,7 @@ void shouldFindJobInfoByUri() { @Test void shouldReturnAbsentStatus() { - DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(),JOBS_TABLE_NAME, 10); + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); } @@ -168,7 +171,7 @@ void shouldFindAll() { @Test void shouldFindAllWithPaging() { // given - testee = new DynamoJobRepository(getDynamoDbClient(),JOBS_TABLE_NAME, 2); + testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest1", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -194,7 +197,7 @@ void shouldFindAllinSizeOperation() { @Test void shouldFindAllinSizeOperationWithPageing() { // given - testee = new DynamoJobRepository(getDynamoDbClient(),JOBS_TABLE_NAME, 2); + testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 2); testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); testee.createOrUpdate(newJobInfo("youn44444556gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); @@ -233,7 +236,6 @@ void shouldFindLatestDistinct() { } - @Test void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() { // given @@ -384,16 +386,18 @@ void shouldUpdateJobLastUpdateTime() { @Test void shouldClearJobInfos() { //Given - JobInfo stoppedJob = builder() - .setJobId("some/job/stopped") - .setJobType("test") - .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) - .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) - .setHostname("localhost") - .setStatus(JobStatus.OK) - .build(); - testee.createOrUpdate(stoppedJob); - + // 25 is the max delete batch size + for (int i = 0; i < 26; i++) { + JobInfo stoppedJob = builder() + .setJobId("some/job/stopped" + i) + .setJobType("test") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build(); + testee.createOrUpdate(stoppedJob); + } //When testee.deleteAll(); @@ -448,4 +452,26 @@ private JobInfo jobInfo(final String jobId, final String type) { "localhost" ); } + + private void createJobInfoTable() { + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(JOBS_TABLE_NAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName(ID.key()) + .attributeType(S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName(ID.key()) + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build()) + .build()); + } + + private void deleteJobInfoTable() { + getDynamoDbClient().deleteTable(DeleteTableRequest.builder().tableName(JOBS_TABLE_NAME).build()); + } } From 04140b06d2846e3619de1acd0018582d2f248c27 Mon Sep 17 00:00:00 2001 From: fw/tm Date: Wed, 15 Jan 2020 16:34:26 +0100 Subject: [PATCH 65/78] =?UTF-8?q?task=20=F0=9F=93=9D|fw/tm|fix=20enabling?= =?UTF-8?q?=20of=20jobs=20and=20clearRunningJobs;=20document=20changes=20i?= =?UTF-8?q?n=20changelog;=20release=20version=202.2.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Falk Woldmann Co-authored-by: Tony Moeller --- CHANGELOG.md | 6 ++++++ build.gradle | 2 +- .../repository/dynamo/DynamoJobMetaRepository.java | 14 ++++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f33ac369f..34ef91e4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Release Notes +## 2.2.3 +* **[edison-jobs]**: DynamoDb Support + * creation of tables removed + * Fixes for Enabling Jobs and ClearRunningJob + * Clear table when calling deleteAll instead of deleting and recreating table + ## 2.2.2 * **[edison-jobs]**: add DynamoDb Support for Edison-Jobs * Properties for enabling DynamoDb: diff --git a/build.gradle b/build.gradle index 909e17def..77fc6f3ad 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.3-SNAPSHOT' + version = '2.2.3' group = 'de.otto.edison' repositories { diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java index ca7cac07c..5d0e1b703 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -11,6 +11,7 @@ import static java.util.Collections.emptyMap; import static java.util.stream.Collectors.toList; +import static software.amazon.awssdk.services.dynamodb.model.AttributeAction.*; public class DynamoJobMetaRepository extends AbstractDynamoRepository implements JobMetaRepository { @@ -72,7 +73,7 @@ public String getRunningJob(String jobType) { @Override public void clearRunningJob(String jobType) { - setValue(jobType, KEY_RUNNING, null); + removeAttribute(jobType, KEY_RUNNING); } @Override @@ -82,7 +83,7 @@ public void disable(String jobType, String comment) { @Override public void enable(String jobType) { - setValue(jobType, KEY_DISABLED, null); + removeAttribute(jobType, KEY_DISABLED); } @Override @@ -91,6 +92,15 @@ public String setValue(String jobType, String key, String value) { return putValue(jobType, key, value); } + private void removeAttribute(String jobType, String attributeKey) { + dynamoDbClient.updateItem(UpdateItemRequest.builder() + .tableName(tableName) + .key(ImmutableMap.of(JOB_TYPE_KEY, AttributeValue.builder().s(jobType).build())) + .attributeUpdates(ImmutableMap.of(attributeKey, AttributeValueUpdate.builder() + .action(DELETE).build())) + .build()); + } + private String putValue(String jobType, String key, String value) { GetItemResponse getItemResponse = getItem(jobType); Map newEntry = new HashMap<>(getItemResponse.item()); From d41a1853f1058a3028f05045c32c8982e4c57d64 Mon Sep 17 00:00:00 2001 From: Frank Bregulla Date: Thu, 16 Jan 2020 09:22:10 +0100 Subject: [PATCH 66/78] replace repository "repo.spring.io" with mavenCentral() and jcenter() because repo.spring.io always returns 403 --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 909e17def..123a0dbdd 100644 --- a/build.gradle +++ b/build.gradle @@ -2,8 +2,8 @@ buildscript { apply from: "${rootDir}/gradle/dependencies.gradle" repositories { - maven { url 'http://repo.spring.io/libs-snapshot' } - maven { url 'http://repo.spring.io/plugins-release' } + mavenCentral() + jcenter() mavenLocal() } From c38df0461c29d55034ca65e42e32623ae39fe848 Mon Sep 17 00:00:00 2001 From: tm Date: Mon, 20 Jan 2020 09:42:15 +0100 Subject: [PATCH 67/78] =?UTF-8?q?task=20=F0=9F=93=9D|tm|fix=20JobMetaRepos?= =?UTF-8?q?itoryTest=20for=20DynamoDb?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Tony Moeller --- .../mongo/JobMetaRepositoryTest.java | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index 484a1ccb8..93b0b2983 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -19,7 +19,7 @@ import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.dynamodb.model.DeleteTableRequest; +import software.amazon.awssdk.services.dynamodb.model.*; import java.io.IOException; import java.net.URI; @@ -48,6 +48,7 @@ public static void teardownMongo() { @BeforeAll public static void initDbs() throws IOException { EmbeddedMongoHelper.startMongoDB(); + createDynamoTableTable(); dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); } @@ -57,11 +58,40 @@ public static void initDbs() throws IOException { @BeforeEach void setUp() { + createDynamoTableTable(); dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); } @AfterEach public void tearDown() { + deleteDynamoTable(); + } + + private static void createDynamoTableTable() { + try { + getDynamoDbClient().describeTable(DescribeTableRequest.builder() + .tableName(DYNAMO_JOB_META_TABLE_NAME) + .build()); + } catch (ResourceNotFoundException e) { + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(DYNAMO_JOB_META_TABLE_NAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName("jobType") + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName("jobType") + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build()); + } + } + + private static void deleteDynamoTable() { DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() .tableName(DYNAMO_JOB_META_TABLE_NAME).build(); getDynamoDbClient().deleteTable(deleteTableRequest); From 60e4656aeb9ec0490b0803a5956a113acb8daff9 Mon Sep 17 00:00:00 2001 From: sm/tm Date: Mon, 20 Jan 2020 10:57:45 +0100 Subject: [PATCH 68/78] =?UTF-8?q?task=20=F0=9F=93=9D|sm/tm|remove=20warnin?= =?UTF-8?q?gs=20for=20dynamo=20implementation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Simon Monecke Co-authored-by: Tony Moeller --- .../jobs/repository/dynamo/DynamoContainer.java | 11 +++++++++++ .../dynamo/DynamoJobMetaRepositoryTest.java | 6 +++++- .../repository/dynamo/DynamoJobRepositoryTest.java | 8 +++++--- .../jobs/repository/mongo/JobMetaRepositoryTest.java | 12 ++++++++---- 4 files changed, 29 insertions(+), 8 deletions(-) create mode 100644 edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java new file mode 100644 index 000000000..4566cf5c1 --- /dev/null +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java @@ -0,0 +1,11 @@ +package de.otto.edison.jobs.repository.dynamo; + +import org.testcontainers.containers.GenericContainer; + +public class DynamoContainer extends GenericContainer { + + public DynamoContainer(final String dockerImageName) { + super(dockerImageName); + } + +} diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java index b26b40cae..1b867008b 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java @@ -29,13 +29,17 @@ public class DynamoJobMetaRepositoryTest { @Container - private static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + private static GenericContainer dynamodb = createTestContainer() .withExposedPorts(8000); private static final String TABLE_NAME = "jobMeta"; private static DynamoJobMetaRepository dynamoJobMetaRepository; + public static GenericContainer createTestContainer() { + return new GenericContainer<>("amazon/dynamodb-local:latest"); + } + @BeforeEach public void before() { createJobInfoTable(); diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java index 307bbc9ef..9f4ffb3da 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -54,13 +54,11 @@ class DynamoJobRepositoryTest { private static DynamoJobRepository testee; @Container - private static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + private static GenericContainer dynamodb = createTestContainer() .withExposedPorts(8000); @AfterEach void tearDown() { - DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() - .tableName(JOBS_TABLE_NAME).build(); deleteJobInfoTable(); } @@ -70,6 +68,10 @@ void setUp() { testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); } + public static GenericContainer createTestContainer() { + return new GenericContainer<>("amazon/dynamodb-local:latest"); + } + private static DynamoDbClient getDynamoDbClient() { String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + dynamodb.getMappedPort(8000); diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index 93b0b2983..2d3bbf57c 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -48,17 +48,17 @@ public static void teardownMongo() { @BeforeAll public static void initDbs() throws IOException { EmbeddedMongoHelper.startMongoDB(); - createDynamoTableTable(); + createDynamoTable(); dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); } @Container - static GenericContainer dynamodb = new GenericContainer("amazon/dynamodb-local:latest") + private static GenericContainer dynamodb = createTestContainer() .withExposedPorts(8000);; @BeforeEach void setUp() { - createDynamoTableTable(); + createDynamoTable(); dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); } @@ -67,7 +67,11 @@ public void tearDown() { deleteDynamoTable(); } - private static void createDynamoTableTable() { + private static GenericContainer createTestContainer() { + return new GenericContainer<>("amazon/dynamodb-local:latest"); + } + + private static void createDynamoTable() { try { getDynamoDbClient().describeTable(DescribeTableRequest.builder() .tableName(DYNAMO_JOB_META_TABLE_NAME) From 7c07b39d3734ed61a6993698bc4f5cecd9c9d470 Mon Sep 17 00:00:00 2001 From: sm/tm Date: Mon, 20 Jan 2020 11:00:36 +0100 Subject: [PATCH 69/78] =?UTF-8?q?task=20=F0=9F=93=9D|sm/tm|remove=20unused?= =?UTF-8?q?=20dynamo=20class?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Simon Monecke Co-authored-by: Tony Moeller --- .../jobs/repository/dynamo/DynamoContainer.java | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java deleted file mode 100644 index 4566cf5c1..000000000 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoContainer.java +++ /dev/null @@ -1,11 +0,0 @@ -package de.otto.edison.jobs.repository.dynamo; - -import org.testcontainers.containers.GenericContainer; - -public class DynamoContainer extends GenericContainer { - - public DynamoContainer(final String dockerImageName) { - super(dockerImageName); - } - -} From 3d0cc0f1cf5e7a156b3098eb6cd60de7efb2f7d5 Mon Sep 17 00:00:00 2001 From: Florian Torkler Date: Tue, 21 Jan 2020 09:24:56 +0100 Subject: [PATCH 70/78] Next snapshot --- CHANGELOG.md | 2 ++ build.gradle | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ef91e4f..d9e261e64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Release Notes +## 2.2.4 - SNAPSHOT + ## 2.2.3 * **[edison-jobs]**: DynamoDb Support * creation of tables removed diff --git a/build.gradle b/build.gradle index 6d5e40b1c..3b0c43c8e 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.3' + version = '2.2.4-SNAPSHOT' group = 'de.otto.edison' repositories { From 1d39d35a2350ddc9f7de4032d2ead6d1f61980ca Mon Sep 17 00:00:00 2001 From: bs/tm Date: Tue, 21 Jan 2020 14:51:45 +0100 Subject: [PATCH 71/78] =?UTF-8?q?task=20=F0=9F=93=9D|bs/tm|next=20fix=20fo?= =?UTF-8?q?r=20DynamoRepository?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bennet Schulz Co-authored-by: Tony Moeller --- .../dynamo/DynamoJobRepository.java | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java index 517690bef..9c942ede6 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -167,20 +167,12 @@ public List findByType(String jobType) { @Override public JobInfo createOrUpdate(final JobInfo job) { - Map jobAsItem = encode(job); - PutItemRequest putItemRequest = PutItemRequest.builder() - .tableName(tableName) - .item(jobAsItem) - .build(); - dynamoDbClient.putItem(putItemRequest); - return job; - } - - public JobInfo createOrUpdate(final JobInfo job, final AttributeValue etag) { Map jobAsItem = encode(job); final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() .tableName(tableName) .item(jobAsItem); + final Map jobInfo = findOneItem(job.getJobId()).orElse(Collections.emptyMap()); + final AttributeValue etag = jobInfo.get(ETAG_KEY); if (etag != null) { Map valueMap = new HashMap<>(); valueMap.put(":val", AttributeValue.builder().s(etag.s()).build()); @@ -204,6 +196,7 @@ private Map encode(JobInfo jobInfo) { jobAsItem.put(LAST_UPDATED_EPOCH.key(), toNumberAttributeValue(jobInfo.getLastUpdated().toInstant().toEpochMilli())); } jobAsItem.put(MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); + jobAsItem.put(ETAG_KEY, toStringAttributeValue(UUID.randomUUID().toString())); return jobAsItem; } @@ -266,12 +259,10 @@ public JobInfo.JobStatus findStatus(String jobId) { public void appendMessage(String jobId, JobMessage jobMessage) { final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); JobInfo jobInfo = decode(item); - createOrUpdate( - jobInfo.copy() + createOrUpdate(jobInfo.copy() .addMessage(jobMessage) .setLastUpdated(jobMessage.getTimestamp()) - .build(), - item.get(ETAG_KEY)); + .build()); } @Override @@ -280,8 +271,7 @@ public void setJobStatus(String jobId, JobInfo.JobStatus jobStatus) { JobInfo jobInfo = decode(item); createOrUpdate(jobInfo.copy() .setStatus(jobStatus) - .build(), - item.get(ETAG_KEY)); + .build()); } @Override @@ -290,8 +280,7 @@ public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { JobInfo jobInfo = decode(item); createOrUpdate(jobInfo.copy() .setLastUpdated(lastUpdate) - .build(), - item.get(ETAG_KEY)); + .build()); } @Override From 61bb074f837cf8866dd0ffc03a4c9a478d0fc449 Mon Sep 17 00:00:00 2001 From: bs/tm Date: Tue, 21 Jan 2020 15:21:19 +0100 Subject: [PATCH 72/78] =?UTF-8?q?task=20=F0=9F=93=9D|bs/tm|bump=20version?= =?UTF-8?q?=20to=202.2.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bennet Schulz Co-authored-by: Tony Moeller --- CHANGELOG.md | 4 ++++ build.gradle | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ef91e4f..152f1885d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release Notes +## 2.2.4 +* **[edison-jobs]**: DynamoDb Support + * fix ETag handling for DynamoJobRepository + ## 2.2.3 * **[edison-jobs]**: DynamoDb Support * creation of tables removed diff --git a/build.gradle b/build.gradle index 77fc6f3ad..2bbc2105a 100644 --- a/build.gradle +++ b/build.gradle @@ -37,7 +37,7 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.3' + version = '2.2.4' group = 'de.otto.edison' repositories { From 57aad5313bce5bb87e52f3778bf0190b219acb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20W=C3=B6lfel?= Date: Tue, 21 Jan 2020 16:01:05 +0100 Subject: [PATCH 73/78] bump next snapshot Co-authored-by: Jan Schawo --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 857055849..c75b0a248 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,7 @@ subprojects { // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.4' + version = '2.2.5-SNAPSHOT' group = 'de.otto.edison' repositories { From d6b28769f80020d95cb47495f1ea07aed62060d5 Mon Sep 17 00:00:00 2001 From: Guido Steinacker Date: Sat, 9 Mar 2019 14:23:39 +0100 Subject: [PATCH 74/78] Fixes thread-safety issue in OAuthPublicKeyInMemoryRepository --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 152f1885d..40fe3ed18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,10 +71,13 @@ Suppress unnecessary warning on startup Block query of public keys until keys have been initially fetched from server ## 2.0.0-rc5 + * **[edison-jobs]**: -Reimplement the JobEventPublisher from Edison 1.x for backwards compatibility + - Reimplement the JobEventPublisher from Edison 1.x for backwards compatibility + * **[edison-oauth]**: -Add dependency to org.springframework.security:spring-security-web:5.1.4.RELEASE + - Add dependency to org.springframework.security:spring-security-web:5.1.4.RELEASE + - Fixes thread-safety issue in OAuthPublicKeyInMemoryRepository ## 2.0.0-rc4 From 4307728703037b19961c8cb49d265e25bd3da6cf Mon Sep 17 00:00:00 2001 From: Guido Steinacker Date: Sat, 9 Mar 2019 17:00:11 +0100 Subject: [PATCH 75/78] Rendering abbrev. commitId instead of the full one --- edison-core/src/main/resources/templates/status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/edison-core/src/main/resources/templates/status.html b/edison-core/src/main/resources/templates/status.html index af5cfb200..11dc3a3c4 100644 --- a/edison-core/src/main/resources/templates/status.html +++ b/edison-core/src/main/resources/templates/status.html @@ -67,7 +67,7 @@

Version

Commit Message:
Short Message
Commit ID:
-
Git commit
+
Git commit
Commit Time:
Git time
Branch:
From c21cca75083faa62508ad621680f7a9b3839441e Mon Sep 17 00:00:00 2001 From: Guido Steinacker Date: Mon, 3 Feb 2020 17:20:38 +0100 Subject: [PATCH 76/78] Fixes bug with respect to cglib not beeing able to generate subclasses for proxies --- .../java/de/otto/edison/status/scheduler/CronScheduler.java | 2 +- .../otto/edison/status/scheduler/EveryTenSecondsScheduler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java b/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java index ada0b7ce5..a25f0378a 100644 --- a/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java +++ b/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java @@ -3,7 +3,7 @@ import de.otto.edison.status.indicator.ApplicationStatusAggregator; import org.springframework.scheduling.annotation.Scheduled; -public final class CronScheduler implements Scheduler { +public class CronScheduler implements Scheduler { private final ApplicationStatusAggregator aggregator; diff --git a/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java b/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java index 3d4673fae..5378ad45c 100644 --- a/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java +++ b/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java @@ -3,7 +3,7 @@ import de.otto.edison.status.indicator.ApplicationStatusAggregator; import org.springframework.scheduling.annotation.Scheduled; -public final class EveryTenSecondsScheduler implements Scheduler{ +public class EveryTenSecondsScheduler implements Scheduler{ private static final int TEN_SECONDS = 10 * 1000; From 875ca2a8726dcc933a68c2d9aaf925a00c8309f6 Mon Sep 17 00:00:00 2001 From: Jan Schawo Date: Tue, 4 Feb 2020 13:36:15 +0100 Subject: [PATCH 77/78] Release version 2.2.5 with latest aws sdk 2.10.56 --- CHANGELOG.md | 3 +++ build.gradle | 2 +- gradle/dependencies.gradle | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40fe3ed18..464813916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Release Notes +## 2.2.5 +* **[general]**: Use latest AWS SDK 2.10.56 as per AWS recommendation (important bugfixes) + ## 2.2.4 * **[edison-jobs]**: DynamoDb Support * fix ETag handling for DynamoJobRepository diff --git a/build.gradle b/build.gradle index c75b0a248..20ff515c1 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,7 @@ subprojects { // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.5-SNAPSHOT' + version = '2.2.5' group = 'de.otto.edison' repositories { diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index a6c9bbe9a..cdebd350c 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -23,7 +23,7 @@ ext { edison_hal : '2.0.2', validator_collection : '2.2.0', slf4j : '1.7.26', - aws_sdk : '2.10.1', + aws_sdk : '2.10.56', java_validation_api : '2.0.1.Final', java_xml : '2.3.0', jackson : '2.10.0' From 6c47d732222bb4725ee2095920d7c360a53cf682 Mon Sep 17 00:00:00 2001 From: Jan Schawo Date: Tue, 4 Feb 2020 13:44:04 +0100 Subject: [PATCH 78/78] Next snapshot version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 20ff515c1..73032b30e 100644 --- a/build.gradle +++ b/build.gradle @@ -38,7 +38,7 @@ subprojects { // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.2.5' + version = '2.2.6-SNAPSHOT' group = 'de.otto.edison' repositories {