diff --git a/.travis.yml b/.travis.yml index 8bdd1c449..ca871d96d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: java jdk: - openjdk8 - - oraclejdk11 + - openjdk11 install: /bin/true # skip gradle assemble diff --git a/CHANGELOG.md b/CHANGELOG.md index be58467a1..464813916 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,56 @@ # Release Notes +## 2.2.5 +* **[general]**: Use latest AWS SDK 2.10.56 as per AWS recommendation (important bugfixes) + +## 2.2.4 +* **[edison-jobs]**: DynamoDb Support + * fix ETag handling for DynamoJobRepository + +## 2.2.3 +* **[edison-jobs]**: DynamoDb Support + * creation of tables removed + * Fixes for Enabling Jobs and ClearRunningJob + * Clear table when calling deleteAll instead of deleting and recreating table + +## 2.2.2 +* **[edison-jobs]**: add DynamoDb Support for Edison-Jobs + * Properties for enabling DynamoDb: + * **edison.jobs.dynamo.enabled**: Enable DynamoDb (disabled by default) + * **edison.jobs.mongo.enabled**: MongoDb needs to be disabled + * **edison.jobs.dynamo.jobinfo.tableName**: Name for JobInfo table (gets created if non-existent) + * **edison.jobs.dynamo.jobinfo.pageSize**: PageSize for scan-requests against JobInfo table + * **edison.jobs.dynamo.jobmeta.tableName**: Name for JobMeta table (gets created if non-existent) + +## 2.2.1 +* **[general]**: upgrade aws sdk +* **[edison-core]**: Fix basic auth credentials retrieval on wrong format + +## 2.2.0 +* **[general]**: Update to Spring Boot 2.2 + +See https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2.2-Release-Notes for a migration guide + +## 2.1.3 +* **[general]**: Update dependencies + +## 2.1.2 +* **[general]**: increase gradle version + +## 2.1.1 +* **[edison-togglz]**: + * Make S3TogglzRepository cache dependent from String instead of Feature in order to support kotlin togglz + * Add `getFeatureFromName(String name)`- function to be able to retrieve current feature by its name + +## 2.1.0 +* **[edison-togglz]**: + * Make FeatureManager @ConditionalOnMissingBean to allow overriding + * Get @Label and other annotations from FeatureManager.getMetaData so that this works with Features that are not enums, too + * Add methods to FeatureManagerSupport that get the FeatureManager as parameter instead of from the FeatureContext + +## 2.0.1 +* **[edison-validation]**: Make error profile configurable via application property 'edison.validation.error-profile' + ## 2.0.0 * **[edison-validation]**: Add EnumListValidator to be able to validate a list of enums @@ -23,10 +74,13 @@ Suppress unnecessary warning on startup Block query of public keys until keys have been initially fetched from server ## 2.0.0-rc5 + * **[edison-jobs]**: -Reimplement the JobEventPublisher from Edison 1.x for backwards compatibility + - Reimplement the JobEventPublisher from Edison 1.x for backwards compatibility + * **[edison-oauth]**: -Add dependency to org.springframework.security:spring-security-web:5.1.4.RELEASE + - Add dependency to org.springframework.security:spring-security-web:5.1.4.RELEASE + - Fixes thread-safety issue in OAuthPublicKeyInMemoryRepository ## 2.0.0-rc4 diff --git a/README.md b/README.md index 9e5c96881..4e2a50e64 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,9 @@ Collection of independent libraries on top of Spring Boot to provide a faster se > "I never did anything by accident, nor did any of my inventions come by accident; they came by work." - Thomas Edison - ## Status -[![Next Selected Stories](https://badge.waffle.io/otto-de/edison-microservice.svg?label=Ready&title=Selected)](http://waffle.io/otto-de/edison-microservice) -[![Active Stories](https://badge.waffle.io/otto-de/edison-microservice.svg?label=In%20Progress&title=Doing)](http://waffle.io/otto-de/edison-microservice) - -[![build](https://travis-ci.org/otto-de/edison-microservice.svg)](https://travis-ci.org/otto-de/edison-microservice) +[![build](https://travis-ci.org/otto-de/edison-microservice.svg?branch=master)](https://travis-ci.org/otto-de/edison-microservice) [![codecov](https://codecov.io/gh/otto-de/edison-microservice/branch/master/graph/badge.svg)](https://codecov.io/gh/otto-de/edison-microservice) [![Known Vulnerabilities](https://snyk.io/test/github/otto-de/edison-microservice/badge.svg)](https://snyk.io/test/github/otto-de/edison-microservice) [![release](https://maven-badges.herokuapp.com/maven-central/de.otto.edison/edison-core/badge.svg)](https://maven-badges.herokuapp.com/maven-central/de.otto.edison/edison-core) @@ -18,7 +14,6 @@ Collection of independent libraries on top of Spring Boot to provide a faster se Have a look at the [release notes](CHANGELOG.md) for details about updates and changes. - ## About This project contains a number of independent libraries on top of Spring Boot to provide a faster setup of jvm microservices. @@ -45,7 +40,7 @@ This project maintains its roadmap with [issues](https://github.com/otto-de/edis **[1.x.0](https://github.com/otto-de/edison-microservice/milestone/2)**: Edison Microservices for Spring Boot 1.5 ✔ -**[2.0.0](https://github.com/otto-de/edison-microservice/milestone/3)**: Edison Microservices for Spring Boot 2.0 +**[2.0.0](https://github.com/otto-de/edison-microservice/milestone/3)**: Edison Microservices for Spring Boot 2.x ## Migration from Edison 1.x to Edison 2 diff --git a/build.gradle b/build.gradle index 63ef9e566..73032b30e 100644 --- a/build.gradle +++ b/build.gradle @@ -2,8 +2,8 @@ buildscript { apply from: "${rootDir}/gradle/dependencies.gradle" repositories { - maven { url 'http://repo.spring.io/libs-snapshot' } - maven { url 'http://repo.spring.io/plugins-release' } + mavenCentral() + jcenter() mavenLocal() } @@ -37,7 +37,8 @@ subprojects { // DO NOT FORGET TO DOCUMENT CHANGES IN CHANGELOG.md // // Add a GitHub release for every new release: https://github.com/otto-de/edison-microservice/releases - version = '2.0.1-SNAPSHOT' + + version = '2.2.6-SNAPSHOT' group = 'de.otto.edison' repositories { @@ -50,6 +51,7 @@ subprojects { // Override some Spring Boot default versions // see https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#howto-customize-dependency-versions ext['mockito.version'] = test_versions.mockito_core + ext['jackson.version'] = versions.jackson task allDeps(type: DependencyReportTask) {} diff --git a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java index 1e7401a2e..7bdb8791f 100644 --- a/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java +++ b/edison-core/src/main/java/de/otto/edison/authentication/Credentials.java @@ -37,13 +37,27 @@ public String getPassword() { */ public static Optional readFrom(HttpServletRequest request) { String authorizationHeader = request.getHeader("Authorization"); - if (!StringUtils.isEmpty(authorizationHeader)) { + if (!StringUtils.isEmpty(authorizationHeader) && authorizationHeader.contains("Basic")) { String credentials = authorizationHeader.substring(6, authorizationHeader.length()); - String[] decodedCredentialParts = new String(Base64Utils.decode(credentials.getBytes())).split(":", 2); - if (!decodedCredentialParts[0].isEmpty() && !decodedCredentialParts[1].isEmpty()) { + Optional decodedCredentials = base64Decode(credentials); + String[] decodedCredentialParts = decodedCredentials + .map(s1 -> s1.split(":", 2)) + .orElse(new String[0]); + if (decodedCredentialParts.length >= 2 + && !decodedCredentialParts[0].isEmpty() + && !decodedCredentialParts[1].isEmpty()) { + return Optional.of(new Credentials(decodedCredentialParts[0], decodedCredentialParts[1])); } } return Optional.empty(); } + + private static Optional base64Decode(String input) { + try { + return Optional.of(new String(Base64Utils.decode(input.getBytes()))); + } catch (IllegalArgumentException e) { + return Optional.empty(); + } + } } diff --git a/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java b/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java index 4108e4498..026519ea0 100644 --- a/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java +++ b/edison-core/src/main/java/de/otto/edison/logging/ui/LoggersHtmlEndpoint.java @@ -132,7 +132,7 @@ private List> getLoggers() { .keySet() .stream() .map(key -> key.contains("$") ? null : new HashMap() {{ - final LoggerLevels logger = (LoggerLevels) loggers.get(key); + final LoggersEndpoint.SingleLoggerLevels logger = (LoggersEndpoint.SingleLoggerLevels) loggers.get(key); put("name", key); put("displayName", displayNameOf(key)); put("configuredLevel", logger.getConfiguredLevel()); diff --git a/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java b/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java index ada0b7ce5..a25f0378a 100644 --- a/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java +++ b/edison-core/src/main/java/de/otto/edison/status/scheduler/CronScheduler.java @@ -3,7 +3,7 @@ import de.otto.edison.status.indicator.ApplicationStatusAggregator; import org.springframework.scheduling.annotation.Scheduled; -public final class CronScheduler implements Scheduler { +public class CronScheduler implements Scheduler { private final ApplicationStatusAggregator aggregator; diff --git a/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java b/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java index 3d4673fae..5378ad45c 100644 --- a/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java +++ b/edison-core/src/main/java/de/otto/edison/status/scheduler/EveryTenSecondsScheduler.java @@ -3,7 +3,7 @@ import de.otto.edison.status.indicator.ApplicationStatusAggregator; import org.springframework.scheduling.annotation.Scheduled; -public final class EveryTenSecondsScheduler implements Scheduler{ +public class EveryTenSecondsScheduler implements Scheduler{ private static final int TEN_SECONDS = 10 * 1000; diff --git a/edison-core/src/main/resources/templates/status.html b/edison-core/src/main/resources/templates/status.html index af5cfb200..11dc3a3c4 100644 --- a/edison-core/src/main/resources/templates/status.html +++ b/edison-core/src/main/resources/templates/status.html @@ -67,7 +67,7 @@

Version

Commit Message:
Short Message
Commit ID:
-
Git commit
+
Git commit
Commit Time:
Git time
Branch:
diff --git a/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java b/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java index 2e6538ff3..2d6eb5af2 100644 --- a/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java +++ b/edison-core/src/test/java/de/otto/edison/acceptance/status/StatusControllerAcceptanceTest.java @@ -44,7 +44,7 @@ public void shouldGetInternalStatusAsMonitoringStatusJson() throws IOException { then( assertThat(the_status_code().value(), is(200)), - assertThat(the_response_headers().get("Content-Type"), contains("application/vnd.otto.monitoring.status+json;charset=UTF-8")) + assertThat(the_response_headers().get("Content-Type"), contains("application/vnd.otto.monitoring.status+json")) ); } diff --git a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java index 58e755f5c..a3cdc721d 100644 --- a/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java +++ b/edison-core/src/test/java/de/otto/edison/authentication/CredentialsTest.java @@ -75,6 +75,20 @@ public void shouldReturnEmptyCredentialsIfUsernameNotSet() { assertThat(credentials.isPresent(), is(false)); } + @Test + public void shouldReturnEmptyCredentialsIfAnotherAuthorizationSchemeThanBasicIsUsed() { + // given + when(httpServletRequest.getHeader("Authorization")) + .thenReturn( + "Token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } + @Test public void shouldReturnCorrectCredentialsIfPasswordContainsColons() { // given @@ -88,4 +102,29 @@ public void shouldReturnCorrectCredentialsIfPasswordContainsColons() { assertThat(credentials.get().getUsername(), is("user")); assertThat(credentials.get().getPassword(), is("pass:word")); } + + @Test + public void shouldReturnEmptyCredentialsIfColonDoesNotExist() { + // given + mockHttpServletRequestWithAuthentication("userpass"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } + + @Test + public void shouldReturnEmptyCredentialsIfAuthenticationNotBasic() { + // given + when(httpServletRequest.getHeader("Authorization")) + .thenReturn("Bearer someToken"); + + // when + final Optional credentials = Credentials.readFrom(httpServletRequest); + + // then + assertThat(credentials.isPresent(), is(false)); + } } \ No newline at end of file diff --git a/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java b/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java index 66fd5ab43..89046823d 100644 --- a/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java +++ b/edison-core/src/test/java/de/otto/edison/logging/ui/DisableEndpointPostProcessorTest.java @@ -46,7 +46,7 @@ public void shouldDisableEndpoint() { @Configuration static class TestEndpointConfiguration { @Bean - Object someTestMvcEndpoint() { + static Object someTestMvcEndpoint() { return new Object(); } } @@ -54,7 +54,7 @@ Object someTestMvcEndpoint() { @Configuration static class RemoveTestEndpointConfiguration { @Bean - DisableEndpointPostProcessor withoutSomeBean() { + static DisableEndpointPostProcessor withoutSomeBean() { return new DisableEndpointPostProcessor("someTest"); } } diff --git a/edison-core/src/test/resources/application.yml b/edison-core/src/test/resources/application.yml index cc16c54aa..65fa40d8d 100644 --- a/edison-core/src/test/resources/application.yml +++ b/edison-core/src/test/resources/application.yml @@ -7,6 +7,8 @@ spring: favor-parameter: true media-types: html: text/html + #jmx: + # enabled: true # context + port of the application server: @@ -14,13 +16,19 @@ server: context-path: /testcore port: 8084 + + # context of the management endpoints like metrics, health, and so on # default is /actuator management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true # edison-specific configuration edison: diff --git a/edison-jobs/build.gradle b/edison-jobs/build.gradle index 96eeae647..3963fe08e 100644 --- a/edison-jobs/build.gradle +++ b/edison-jobs/build.gradle @@ -7,6 +7,9 @@ dependencies { implementation libraries.mongodb_driver implementation project(":edison-mongo") + implementation libraries.aws_sdk_dynamodb + + implementation project(":edison-core") implementation group: 'io.micrometer', name: 'micrometer-core', version: '1.2.0' implementation libraries.jcip_annotations @@ -20,6 +23,8 @@ dependencies { testImplementation test_libraries.json_path testImplementation test_libraries.jsonassert testImplementation test_libraries.embedded_mongo + testImplementation test_libraries.testcontainers + } artifacts { diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java b/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java new file mode 100644 index 000000000..8be454000 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/configuration/DynamoJobsConfiguration.java @@ -0,0 +1,43 @@ +package de.otto.edison.jobs.configuration; + +import de.otto.edison.jobs.repository.JobMetaRepository; +import de.otto.edison.jobs.repository.JobRepository; +import de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository; +import de.otto.edison.jobs.repository.dynamo.DynamoJobRepository; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; + +import static org.slf4j.LoggerFactory.getLogger; + +@Configuration +@ConditionalOnProperty(prefix = "edison.jobs", name = "dynamo.enabled", havingValue = "true") +@ConditionalOnBean(type = "software.amazon.awssdk.services.dynamodb.DynamoDbClient") +public class DynamoJobsConfiguration { + + private static final Logger LOG = getLogger(DynamoJobsConfiguration.class); + + @Bean + public JobRepository jobRepository(final DynamoDbClient dynamoDbClient, + final @Value("${edison.jobs.dynamo.jobinfo.tableName}") String tableName, + final @Value("${edison.jobs.dynamo.jobinfo.pageSize}") int pageSize) { + LOG.info("==============================="); + LOG.info("Using DynamoJobRepository with tableName {} and pageSize {}.",tableName, pageSize); + LOG.info("==============================="); + return new DynamoJobRepository(dynamoDbClient, tableName, pageSize); + } + + @Bean + public JobMetaRepository jobMetaRepository(final DynamoDbClient dynamoDbClient, + final @Value("${edison.jobs.dynamo.jobmeta.tableName}") String tableName) { + LOG.info("==============================="); + LOG.info("Using DynamoJobMetaRepository with tableName {}.", tableName); + LOG.info("==============================="); + return new DynamoJobMetaRepository(dynamoDbClient, tableName); + } + +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java b/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java index 6656f2b10..49fa3e0aa 100644 --- a/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/definition/DefaultJobDefinition.java @@ -43,6 +43,27 @@ public static JobDefinition manuallyTriggerableJobDefinition(final String jobTyp return new DefaultJobDefinition(jobType, jobName, description, maxAge, Optional.empty(), Optional.empty(), restarts, 0, Optional.empty()); } + /** + * Create a JobDefinition for a job that will not be triggered automatically by a job trigger. + * + * @param jobType The type of the Job + * @param jobName A human readable name of the Job + * @param description A short description of the job's purpose + * @param restarts The number of restarts if the job failed because of errors or exceptions + * @param retries Specifies how often a job trigger should retry to start the job if triggering fails for some reason. + * @param maxAge Optional maximum age of a job. When the job is not run for longer than this duration, + * a warning is displayed on the status page + * @return JobDefinition + */ + public static JobDefinition manuallyTriggerableJobDefinition(final String jobType, + final String jobName, + final String description, + final int restarts, + final int retries, + final Optional maxAge) { + return new DefaultJobDefinition(jobType, jobName, description, maxAge, Optional.empty(), Optional.empty(), restarts, retries, Optional.empty()); + } + /** * Create a JobDefinition that is using a cron expression to specify, when and how often the job should be triggered. * diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java new file mode 100644 index 000000000..6e3f5f8a7 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/AbstractDynamoRepository.java @@ -0,0 +1,36 @@ +package de.otto.edison.jobs.repository.dynamo; + +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.BatchWriteItemRequest; +import software.amazon.awssdk.services.dynamodb.model.WriteRequest; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.util.Collection; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +abstract class AbstractDynamoRepository { + + final DynamoDbClient dynamoDbClient; + final String tableName; + + AbstractDynamoRepository(DynamoDbClient dynamoDbClient, String tableName) { + this.dynamoDbClient = dynamoDbClient; + this.tableName = tableName; + } + + void deleteEntriesPerBatch(List deleteRequests) { + final int chunkSize = 25; + final AtomicInteger counter = new AtomicInteger(); + final Collection> deleteRequestsSplittedByChunkSize = deleteRequests.stream() + .collect(Collectors.groupingBy(it -> counter.getAndIncrement() / chunkSize)) + .values(); + + deleteRequestsSplittedByChunkSize.forEach + (currentDeleteRequests -> dynamoDbClient.batchWriteItem( + BatchWriteItemRequest.builder().requestItems( + ImmutableMap.of(tableName, currentDeleteRequests)).build())); + + } +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java new file mode 100644 index 000000000..5d0e1b703 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepository.java @@ -0,0 +1,190 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobMeta; +import de.otto.edison.jobs.repository.JobMetaRepository; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toList; +import static software.amazon.awssdk.services.dynamodb.model.AttributeAction.*; + +public class DynamoJobMetaRepository extends AbstractDynamoRepository implements JobMetaRepository { + + private static final String KEY_PREFIX = "_e_"; + static final String KEY_DISABLED = KEY_PREFIX + "disabled"; + private static final String KEY_RUNNING = KEY_PREFIX + "running"; + static final String JOB_TYPE_KEY = "jobType"; + private static final String ETAG_KEY = "etag"; + + public DynamoJobMetaRepository(final DynamoDbClient dynamoDbClient, final String tableName) { + super(dynamoDbClient, tableName); + dynamoDbClient.describeTable(DescribeTableRequest.builder() + .tableName(tableName) + .build()); + } + + @Override + public JobMeta getJobMeta(String jobType) { + GetItemResponse response = getItem(jobType); + Map responseItem = response.item(); + if (!responseItem.isEmpty()) { + final boolean isRunning = responseItem.containsKey(KEY_RUNNING); + final boolean isDisabled = responseItem.containsKey(KEY_DISABLED); + final AttributeValue attributeValueComment = responseItem.get(KEY_DISABLED); + String comment = null; + if (attributeValueComment != null) { + comment = attributeValueComment.s(); + } + Map metaMap = responseItem.entrySet().stream() + .filter(e -> !e.getKey().startsWith(KEY_PREFIX)) + .filter(e -> !e.getKey().equals(JOB_TYPE_KEY)) + .filter(e -> !e.getKey().equals(ETAG_KEY)) + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().s())); + return new JobMeta(jobType, isRunning, isDisabled, comment, metaMap); + } else { + return new JobMeta(jobType, false, false, "", emptyMap()); + } + } + + @Override + public boolean createValue(String jobType, String key, String value) { + if (getValue(jobType, key) == null) { + setValue(jobType, key, value); + return true; + } else { + return false; + } + } + + @Override + public boolean setRunningJob(String jobType, String jobId) { + return createValue(jobType, KEY_RUNNING, jobId); + } + + @Override + public String getRunningJob(String jobType) { + return getValue(jobType, KEY_RUNNING); + } + + @Override + public void clearRunningJob(String jobType) { + removeAttribute(jobType, KEY_RUNNING); + } + + @Override + public void disable(String jobType, String comment) { + setValue(jobType, KEY_DISABLED, comment); + } + + @Override + public void enable(String jobType) { + removeAttribute(jobType, KEY_DISABLED); + } + + @Override + public String setValue(String jobType, String key, String value) { + putIfAbsent(jobType); + return putValue(jobType, key, value); + } + + private void removeAttribute(String jobType, String attributeKey) { + dynamoDbClient.updateItem(UpdateItemRequest.builder() + .tableName(tableName) + .key(ImmutableMap.of(JOB_TYPE_KEY, AttributeValue.builder().s(jobType).build())) + .attributeUpdates(ImmutableMap.of(attributeKey, AttributeValueUpdate.builder() + .action(DELETE).build())) + .build()); + } + + private String putValue(String jobType, String key, String value) { + GetItemResponse getItemResponse = getItem(jobType); + Map newEntry = new HashMap<>(getItemResponse.item()); + newEntry.put(key, toAttributeValue(value)); + newEntry.put(ETAG_KEY, AttributeValue.builder().s(UUID.randomUUID().toString()).build()); + + final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() + .tableName(tableName) + .item(newEntry); + addEtagCondition(putItemRequestBuilder, getItemResponse); + dynamoDbClient.putItem(putItemRequestBuilder.build()); + + AttributeValue existingValueForKey = getItemResponse.item().get(key); + return existingValueForKey == null ? null : existingValueForKey.s(); + } + + private void addEtagCondition(final PutItemRequest.Builder putItemRequestBuilder, final GetItemResponse getItemResponse) { + final AttributeValue existingEtag = getItemResponse.item().get(ETAG_KEY); + if (existingEtag != null) { + Map valueMap = new HashMap<>(); + valueMap.put(":val", AttributeValue.builder().s(existingEtag.s()).build()); + putItemRequestBuilder.expressionAttributeValues(valueMap); + putItemRequestBuilder.conditionExpression("contains(etag, :val)"); + } + } + + private AttributeValue toAttributeValue(String value) { + return value == null || value.isEmpty() ? AttributeValue.builder().nul(true).build() : AttributeValue.builder().s(value).build(); + } + + private void putIfAbsent(String jobType) { + GetItemResponse response = getItem(jobType); + if (response.item().isEmpty()) { + Map item = new HashMap<>(); + item.put(JOB_TYPE_KEY, toAttributeValue(jobType)); + PutItemRequest putItemRequest = PutItemRequest.builder() + .tableName(tableName) + .item(item) + .build(); + dynamoDbClient.putItem(putItemRequest); + } + } + + @Override + public String getValue(String jobType, String key) { + GetItemResponse response = getItem(jobType); + AttributeValue value = response.item().get(key); + if (value != null) { + return value.s(); + } else { + return null; + } + } + + @Override + public Set findAllJobTypes() { + ScanRequest scanRequest = ScanRequest.builder() + .tableName(tableName) + .attributesToGet(JOB_TYPE_KEY).build(); + ScanResponse scanResponse = dynamoDbClient.scan(scanRequest); + Set jobTypes = scanResponse.items().stream().map(m -> m.get(JOB_TYPE_KEY).s()).collect(Collectors.toSet()); + return jobTypes; + } + + @Override + public void deleteAll() { + final List deleteRequests = findAllJobTypes().stream() + .map(jobId -> WriteRequest.builder() + .deleteRequest( + DeleteRequest.builder() + .key(ImmutableMap.of(JOB_TYPE_KEY, AttributeValue.builder().s(jobId).build())) + .build() + ).build() + ).collect(toList()); + + deleteEntriesPerBatch(deleteRequests); + } + + private GetItemResponse getItem(String jobType) { + ImmutableMap itemRequestKey = ImmutableMap.of(JOB_TYPE_KEY, toAttributeValue(jobType)); + GetItemRequest itemRequest = GetItemRequest.builder() + .tableName(tableName) + .key(itemRequestKey) + .build(); + return dynamoDbClient.getItem(itemRequest); + } +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java new file mode 100644 index 000000000..9c942ede6 --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepository.java @@ -0,0 +1,351 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobInfo; +import de.otto.edison.jobs.domain.JobMessage; +import de.otto.edison.jobs.domain.Level; +import de.otto.edison.jobs.repository.JobRepository; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.time.OffsetDateTime; +import java.util.*; +import java.util.stream.Collectors; + +import static de.otto.edison.jobs.repository.dynamo.JobStructure.*; +import static java.util.Collections.emptyList; +import static java.util.Comparator.comparingLong; +import static java.util.stream.Collectors.*; + +public class DynamoJobRepository extends AbstractDynamoRepository implements JobRepository { + + private static final String ETAG_KEY = "etag"; + private final int pageSize; + + public DynamoJobRepository(DynamoDbClient dynamoDbClient, final String tableName, int pageSize) { + super(dynamoDbClient, tableName); + this.pageSize = pageSize; + dynamoDbClient.describeTable(DescribeTableRequest.builder() + .tableName(tableName) + .build()); + } + + @Override + public Optional findOne(String jobId) { + return findOneItem(jobId).map(this::decode); + } + + private Optional> findOneItem(final String jobId) { + Map keyMap = new HashMap<>(); + keyMap.put(ID.key(), toStringAttributeValue(jobId)); + GetItemRequest jobInfoRequest = GetItemRequest.builder() + .tableName(tableName) + .key(keyMap) + .build(); + final GetItemResponse jobInfoResponse = dynamoDbClient.getItem(jobInfoRequest); + if (jobInfoResponse.item().isEmpty()) { + return Optional.empty(); + } + return Optional.of(jobInfoResponse.item()); + } + + @Override + public List findLatest(int maxCount) { + return findAll().stream() + .sorted(Comparator.comparingLong(jobInfo -> jobInfo.getStarted().toInstant().toEpochMilli()).reversed()) + .limit(maxCount) + .collect(toList()); + } + + @Override + public List findLatestJobsDistinct() { + return findAll().stream().collect( + groupingBy( + JobInfo::getJobType, + maxBy(comparingLong(jobInfo -> jobInfo.getLastUpdated().toInstant().toEpochMilli())) + )) + .values().stream() + .filter(Optional::isPresent) + .map(Optional::get) + .collect(toList()); + } + + @Override + public List findLatestBy(String type, int maxCount) { + return findByType(type).stream() + .sorted(Comparator.comparingLong(jobInfo -> + jobInfo.getStarted().toInstant().toEpochMilli()).reversed()) + .limit(maxCount) + .collect(toList()); + } + + @Override + public List findRunningWithoutUpdateSince(OffsetDateTime timeOffset) { + Map lastKeyEvaluated = null; + List jobs = new ArrayList<>(); + Map expressionAttributeValues = ImmutableMap.of( + ":val", AttributeValue.builder().n(String.valueOf(timeOffset.toInstant().toEpochMilli())).build() + ); + do { + final ScanRequest query = ScanRequest.builder() + .tableName(tableName) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated) + .expressionAttributeValues(expressionAttributeValues) + .filterExpression(LAST_UPDATED_EPOCH.key() + " < :val and attribute_not_exists(" + STOPPED.key() + ")") + .build(); + + final ScanResponse response = dynamoDbClient.scan(query); + lastKeyEvaluated = response.lastEvaluatedKey(); + List newJobsFromThisPage = response.items().stream().map(this::decode).collect(toList()); + jobs.addAll(newJobsFromThisPage); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + return jobs; + } + + private List findAll(final boolean withMessages) { + Map lastKeyEvaluated = null; + List jobs = new ArrayList<>(); + do { + final ScanRequest.Builder findAllRequestBuilder = ScanRequest.builder() + .tableName(tableName) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated); + + if (!withMessages) { + String projectionExpressionBuilder = ID.key() + + ", " + STARTED.key() + + ", " + STOPPED.key() + + ", " + JOB_TYPE.key() + + ", #" + STATUS.key() + + ", " + HOSTNAME.key() + + ", " + LAST_UPDATED.key(); + findAllRequestBuilder.projectionExpression(projectionExpressionBuilder) + .expressionAttributeNames(ImmutableMap.of("#" + STATUS.key(), STATUS.key())); + } + final ScanResponse scan = dynamoDbClient.scan(findAllRequestBuilder.build()); + lastKeyEvaluated = scan.lastEvaluatedKey(); + List newJobsFromThisPage = scan.items().stream().map(this::decode).collect(Collectors.toList()); + jobs.addAll(newJobsFromThisPage); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + return jobs; + } + + @Override + public List findAll() { + return findAll(true); + } + + @Override + public List findAllJobInfoWithoutMessages() { + return findAll(false); + } + + @Override + public List findByType(String jobType) { + Map lastKeyEvaluated = null; + List jobs = new ArrayList<>(); + Map expressionAttributeValues = ImmutableMap.of( + ":jobType", AttributeValue.builder().s(jobType).build() + ); + do { + final ScanRequest query = ScanRequest.builder() + .tableName(tableName) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated) + .expressionAttributeValues(expressionAttributeValues) + .filterExpression(JOB_TYPE.key() + " = :jobType") + .build(); + + final ScanResponse response = dynamoDbClient.scan(query); + lastKeyEvaluated = response.lastEvaluatedKey(); + List newJobsFromThisPage = response.items().stream().map(this::decode).collect(toList()); + jobs.addAll(newJobsFromThisPage); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + return jobs; + } + + @Override + public JobInfo createOrUpdate(final JobInfo job) { + Map jobAsItem = encode(job); + final PutItemRequest.Builder putItemRequestBuilder = PutItemRequest.builder() + .tableName(tableName) + .item(jobAsItem); + final Map jobInfo = findOneItem(job.getJobId()).orElse(Collections.emptyMap()); + final AttributeValue etag = jobInfo.get(ETAG_KEY); + if (etag != null) { + Map valueMap = new HashMap<>(); + valueMap.put(":val", AttributeValue.builder().s(etag.s()).build()); + putItemRequestBuilder.expressionAttributeValues(valueMap); + putItemRequestBuilder.conditionExpression("contains(etag, :val)"); + } + dynamoDbClient.putItem(putItemRequestBuilder.build()); + return job; + } + + private Map encode(JobInfo jobInfo) { + Map jobAsItem = new HashMap<>(); + jobAsItem.put(ID.key(), toStringAttributeValue(jobInfo.getJobId())); + jobAsItem.put(HOSTNAME.key(), toStringAttributeValue(jobInfo.getHostname())); + jobAsItem.put(JOB_TYPE.key(), toStringAttributeValue(jobInfo.getJobType())); + jobAsItem.put(STARTED.key(), toStringAttributeValue(jobInfo.getStarted())); + jobAsItem.put(STATUS.key(), toStringAttributeValue(jobInfo.getStatus().name())); + jobInfo.getStopped().ifPresent(offsetDateTime -> jobAsItem.put(STOPPED.key(), toStringAttributeValue(offsetDateTime))); + if (null != jobInfo.getLastUpdated()) { + jobAsItem.put(LAST_UPDATED.key(), toStringAttributeValue(jobInfo.getLastUpdated())); + jobAsItem.put(LAST_UPDATED_EPOCH.key(), toNumberAttributeValue(jobInfo.getLastUpdated().toInstant().toEpochMilli())); + } + jobAsItem.put(MESSAGES.key(), messagesToAttributeValueList(jobInfo.getMessages())); + jobAsItem.put(ETAG_KEY, toStringAttributeValue(UUID.randomUUID().toString())); + return jobAsItem; + } + + private JobInfo decode(Map item) { + final JobInfo.Builder jobInfo = JobInfo.builder() + .setJobId(item.get(ID.key()).s()) + .setHostname(item.get(HOSTNAME.key()).s()) + .setJobType(item.get(JOB_TYPE.key()).s()) + .setStarted(OffsetDateTime.parse(item.get(STARTED.key()).s())) + .setStatus(JobInfo.JobStatus.valueOf(item.get(STATUS.key()).s())) + .setMessages(itemToJobMessages(item)); + if (item.containsKey(STOPPED.key())) { + jobInfo.setStopped(OffsetDateTime.parse(item.get(STOPPED.key()).s())); + } + if (item.containsKey(LAST_UPDATED.key())) { + jobInfo.setLastUpdated(OffsetDateTime.parse(item.get(LAST_UPDATED.key()).s())); + } + return jobInfo.build(); + } + + private List itemToJobMessages(Map item) { + if (!item.containsKey(MESSAGES.key())) { + return emptyList(); + } + final AttributeValue attributeValue = item.get(MESSAGES.key()); + return attributeValue.l().stream().map(this::attributeValueToMessage).collect(toList()); + } + + private JobMessage attributeValueToMessage(AttributeValue attributeValue) { + final Map messageMap = attributeValue.m(); + final Level level = Level.ofKey(messageMap.get(MSG_LEVEL.key()).s()); + final String text = messageMap.get(MSG_TEXT.key()).s(); + final OffsetDateTime timestamp = OffsetDateTime.parse(messageMap.get(MSG_TS.key()).s()); + return JobMessage.jobMessage(level, text, timestamp); + } + + @Override + public void removeIfStopped(String jobId) { + findOne(jobId).ifPresent(jobInfo -> { + if (jobInfo.isStopped()) { + Map keyMap = new HashMap<>(); + keyMap.put(ID.key(), toStringAttributeValue(jobId)); + DeleteItemRequest deleteJobRequest = DeleteItemRequest.builder() + .tableName(tableName) + .key(keyMap) + .build(); + dynamoDbClient.deleteItem(deleteJobRequest); + } + }); + } + + @Override + public JobInfo.JobStatus findStatus(String jobId) { + return findOne(jobId) + .orElseThrow(RuntimeException::new) + .getStatus(); + } + + @Override + public void appendMessage(String jobId, JobMessage jobMessage) { + final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); + JobInfo jobInfo = decode(item); + createOrUpdate(jobInfo.copy() + .addMessage(jobMessage) + .setLastUpdated(jobMessage.getTimestamp()) + .build()); + } + + @Override + public void setJobStatus(String jobId, JobInfo.JobStatus jobStatus) { + final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); + JobInfo jobInfo = decode(item); + createOrUpdate(jobInfo.copy() + .setStatus(jobStatus) + .build()); + } + + @Override + public void setLastUpdate(String jobId, OffsetDateTime lastUpdate) { + final Map item = findOneItem(jobId).orElseThrow(RuntimeException::new); + JobInfo jobInfo = decode(item); + createOrUpdate(jobInfo.copy() + .setLastUpdated(lastUpdate) + .build()); + } + + @Override + public long size() { + Map lastKeyEvaluated = null; + long count = 0; + do { + ScanRequest counterQuery = ScanRequest.builder() + .tableName(tableName) + .select(Select.COUNT) + .limit(pageSize) + .exclusiveStartKey(lastKeyEvaluated) + .build(); + + final ScanResponse countResponse = dynamoDbClient.scan(counterQuery); + lastKeyEvaluated = countResponse.lastEvaluatedKey(); + count = count + countResponse.count(); + } while (lastKeyEvaluated != null && lastKeyEvaluated.size() > 0); + return count; + } + + @Override + public void deleteAll() { + final List deleteRequests = findAll().stream() + .map(JobInfo::getJobId) + .map(jobId -> WriteRequest.builder() + .deleteRequest( + DeleteRequest.builder() + .key(ImmutableMap.of(ID.key(), AttributeValue.builder().s(jobId).build())) + .build() + ).build() + ).collect(toList()); + + deleteEntriesPerBatch(deleteRequests); + } + + private AttributeValue toStringAttributeValue(OffsetDateTime value) { + return toStringAttributeValue(value.toString()); + } + + private AttributeValue toStringAttributeValue(String value) { + return AttributeValue.builder().s(value).build(); + } + + private AttributeValue toNumberAttributeValue(long value) { + return AttributeValue.builder().n(String.valueOf(value)).build(); + } + + private AttributeValue toMapAttributeValue(JobMessage jobMessage) { + Map message = new HashMap<>(); + message.put(MSG_LEVEL.key(), toStringAttributeValue(jobMessage.getLevel().getKey())); + message.put(MSG_TEXT.key(), toStringAttributeValue(jobMessage.getMessage())); + message.put(MSG_TS.key(), toStringAttributeValue(jobMessage.getTimestamp())); + return AttributeValue.builder() + .m(message) + .build(); + } + + private AttributeValue messagesToAttributeValueList(List jobeMessages) { + final List messageAttributes = jobeMessages.stream().map(this::toMapAttributeValue).collect(toList()); + return toAttributeValueList(messageAttributes); + } + + private AttributeValue toAttributeValueList(List values) { + return AttributeValue.builder().l(values).build(); + } + +} diff --git a/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java new file mode 100644 index 000000000..eea3cf14b --- /dev/null +++ b/edison-jobs/src/main/java/de/otto/edison/jobs/repository/dynamo/JobStructure.java @@ -0,0 +1,32 @@ +package de.otto.edison.jobs.repository.dynamo; + +enum JobStructure { + + ID("jobId"), + STARTED("started"), + STOPPED("stopped"), + JOB_TYPE("jobType"), + STATUS("status"), + MESSAGES("messages"), + MSG_TS("ts"), + MSG_TEXT("msg"), + MSG_LEVEL("level"), + HOSTNAME("hostname"), + LAST_UPDATED("lastUpdated"), + LAST_UPDATED_EPOCH("lastUpdatedEpoch"); + + private final String key; + + JobStructure(final String key) { + this.key = key; + } + + public String key() { + return key; + } + + public String toString() { + return key; + } + +} diff --git a/edison-jobs/src/main/resources/static/internal/js/jobs.js b/edison-jobs/src/main/resources/static/internal/js/jobs.js index 98943108f..4d1331e4a 100644 --- a/edison-jobs/src/main/resources/static/internal/js/jobs.js +++ b/edison-jobs/src/main/resources/static/internal/js/jobs.js @@ -11,8 +11,8 @@ function update() { type: "GET", url: jobsUrl + "?humanReadable=true" + (typeFilter === '' ? '' : "&type=" + typeFilter), headers: { - Accept: "application/json; charset=utf-8", - "Content-Type": "application/json; charset=utf-8" + Accept: "application/json", + "Content-Type": "application/json" }, data: {}, dataType: "json", diff --git a/edison-jobs/src/main/resources/static/internal/js/logLoader.js b/edison-jobs/src/main/resources/static/internal/js/logLoader.js index 7cb94a847..33995a799 100644 --- a/edison-jobs/src/main/resources/static/internal/js/logLoader.js +++ b/edison-jobs/src/main/resources/static/internal/js/logLoader.js @@ -3,8 +3,8 @@ function getLog(logIndex) { type: "GET", url: $('.logWindow').data("job-url"), headers: { - Accept: "application/json; charset=utf-8", - "Content-Type": "application/json; charset=utf-8" + Accept: "application/json", + "Content-Type": "application/json" }, data: {}, dataType: "json", diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java new file mode 100644 index 000000000..1b867008b --- /dev/null +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobMetaRepositoryTest.java @@ -0,0 +1,299 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobMeta; +import org.hamcrest.MatcherAssert; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; +import software.amazon.awssdk.utils.ImmutableMap; + +import java.net.URI; +import java.util.Collections; +import java.util.Set; + +import static de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository.JOB_TYPE_KEY; +import static de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository.KEY_DISABLED; +import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertThat; + +@Testcontainers +public class DynamoJobMetaRepositoryTest { + + @Container + private static GenericContainer dynamodb = createTestContainer() + .withExposedPorts(8000); + + private static final String TABLE_NAME = "jobMeta"; + + private static DynamoJobMetaRepository dynamoJobMetaRepository; + + public static GenericContainer createTestContainer() { + return new GenericContainer<>("amazon/dynamodb-local:latest"); + } + + @BeforeEach + public void before() { + createJobInfoTable(); + dynamoJobMetaRepository = new DynamoJobMetaRepository(getDynamoDbClient(), TABLE_NAME); + } + + private static DynamoDbClient getDynamoDbClient() { + String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + + dynamodb.getMappedPort(8000); + + return DynamoDbClient.builder() + .endpointOverride(URI.create(endpointUri)) + .region(Region.EU_CENTRAL_1) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create("acc", "sec"))).build(); + } + + @AfterEach + void tearDown() { + deleteJobInfoTable(); + } + + private void createJobInfoTable() { + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(TABLE_NAME) + .keySchema( + KeySchemaElement.builder().attributeName(JOB_TYPE_KEY).keyType(KeyType.HASH).build() + ) + .attributeDefinitions( + AttributeDefinition.builder().attributeName(JOB_TYPE_KEY).attributeType(ScalarAttributeType.S).build() + ) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build()) + .build()); + } + + private void deleteJobInfoTable() { + getDynamoDbClient().deleteTable(DeleteTableRequest.builder().tableName(TABLE_NAME).build()); + } + + @Test + void shouldSetRunningJob() { + //when + boolean notRunning = dynamoJobMetaRepository.setRunningJob("myJobType", "myJobId"); + + + //given + String jobId = dynamoJobMetaRepository.getRunningJob("myJobType"); + assertThat(jobId, is("myJobId")); + assertThat(notRunning, is(true)); + } + + @Test + public void shouldNotSetRunningWhenIsAlreadyRunning() { + //given + dynamoJobMetaRepository.setRunningJob("myJobType", "otherJobId"); + + //when + boolean notRunning = dynamoJobMetaRepository.setRunningJob("myJobType", "myJobId"); + + //given + String jobId = dynamoJobMetaRepository.getRunningJob("myJobType"); + assertThat(jobId, is("otherJobId")); + assertThat(notRunning, is(false)); + } + + @Test + void shouldSetAndGetValue() { + //given + + //when + dynamoJobMetaRepository.setValue("myJobType", "someKey", "someValue"); + + //given + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, is("someValue")); + } + + @Test + public void shouldReturnPreviousValueOnSetValue() { + //given + dynamoJobMetaRepository.setValue("myJobType", "someKey", "someOldValue"); + + //when + String previousValue = dynamoJobMetaRepository.setValue("myJobType", "someKey", "someNewValue"); + + //given + assertThat(previousValue, is("someOldValue")); + } + + @Test + public void shouldRemoveKeyWhenValueIsNull() { + //given + dynamoJobMetaRepository.setValue("myJobType", "someKey", "someOldValue"); + + //when + String previousValue = dynamoJobMetaRepository.setValue("myJobType", "someKey", null); + + //given + assertThat(previousValue, is("someOldValue")); + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, nullValue()); + } + + @Test + public void createValueShouldAddKeyIfNotExists() { + + //when + boolean valueCreated = dynamoJobMetaRepository.createValue("myJobType", "someKey", "someValue"); + + //then + assertThat(valueCreated, is(true)); + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, is("someValue")); + } + + @Test + public void createValueShouldNotAddKeyIfAlreadyExists() { + //given + dynamoJobMetaRepository.createValue("myJobType", "someKey", "someExistingValue"); + + //when + boolean valueCreated = dynamoJobMetaRepository.createValue("myJobType", "someKey", "someOtherValue"); + + //then + assertThat(valueCreated, is(false)); + String value = dynamoJobMetaRepository.getValue("myJobType", "someKey"); + assertThat(value, is("someExistingValue")); + } + + @Test + public void shouldSetDisabledWithComment() { + //when + dynamoJobMetaRepository.disable("someJobType", "someComment"); + + //then + String value = dynamoJobMetaRepository.getValue("someJobType", KEY_DISABLED); + assertThat(value, is("someComment")); + } + + @Test + public void shouldSetDisabledWithoutComment() { + //when + dynamoJobMetaRepository = new DynamoJobMetaRepository(getDynamoDbClient(), TABLE_NAME); + dynamoJobMetaRepository.disable("someJobType", null); + + //then + String value = dynamoJobMetaRepository.getValue("someJobType", KEY_DISABLED); + assertThat(value, nullValue()); + } + + @Test + public void shouldSetEnabled() { + //given + dynamoJobMetaRepository.disable("someJobType", "disabled"); + + //when + dynamoJobMetaRepository.enable("someJobType"); + + //then + String value = dynamoJobMetaRepository.getValue("someJobType", KEY_DISABLED); + assertThat(value, nullValue()); + } + + @Test + public void shouldClearRunningJob() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId"); + + //when + dynamoJobMetaRepository.clearRunningJob("someJobType"); + + //then + String jobId = dynamoJobMetaRepository.getRunningJob("someJobType"); + assertThat(jobId, nullValue()); + } + + @Test + public void shouldFindAllJobTypes() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId1"); + dynamoJobMetaRepository.setRunningJob("someOtherJobType", "someJobId2"); + dynamoJobMetaRepository.enable("oneMoreJobType"); + + //when + Set allJobTypes = dynamoJobMetaRepository.findAllJobTypes(); + + //then + assertThat(allJobTypes, contains("someJobType", "someOtherJobType", "oneMoreJobType")); + } + + @Test + public void shouldReturnJobMeta() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId"); + dynamoJobMetaRepository.disable("someJobType", "because"); + dynamoJobMetaRepository.setValue("someJobType", "foo", "bar"); + + //when + JobMeta jobMeta = dynamoJobMetaRepository.getJobMeta("someJobType"); + + //then + assertThat(jobMeta.getJobType(), is("someJobType")); + assertThat(jobMeta.isDisabled(), is(true)); + assertThat(jobMeta.getDisabledComment(), is("because")); + assertThat(jobMeta.isRunning(), is(true)); + assertThat(jobMeta.getAll(), is(ImmutableMap.of("foo", "bar"))); + } + + @Test + public void shouldSetDisabledCommentToEmptyStringWhenEnabled() { + //given + dynamoJobMetaRepository.setRunningJob("someJobType", "someJobId"); + dynamoJobMetaRepository.setValue("someJobType", "foo", "bar"); + + //when + JobMeta jobMeta = dynamoJobMetaRepository.getJobMeta("someJobType"); + + //then + assertThat(jobMeta.getJobType(), is("someJobType")); + assertThat(jobMeta.isDisabled(), is(false)); + assertThat(jobMeta.getDisabledComment(), is("")); + } + + @Test + public void shouldReturnEmptyJobMetaWhenJobTypeDoesNotExist() { + //when + JobMeta jobMeta = dynamoJobMetaRepository.getJobMeta("someJobType"); + + //then + assertThat(jobMeta.getJobType(), is("someJobType")); + assertThat(jobMeta.isDisabled(), is(false)); + assertThat(jobMeta.getDisabledComment(), is("")); + assertThat(jobMeta.isRunning(), is(false)); + assertThat(jobMeta.getAll(), is(Collections.emptyMap())); + } + + @Test + void shouldDeleteAll() { + //given + // 25 is the max delete batch size + for (int i = 0; i < 26; i++) { + String jobType = "someJobType" + i; + String key = "someKey" + i; + String value = "someValue" + i; + dynamoJobMetaRepository.createValue(jobType, key, value); + } + + //when + dynamoJobMetaRepository.deleteAll(); + + //then + MatcherAssert.assertThat(dynamoJobMetaRepository.findAllJobTypes(), is(emptySet())); + } +} \ No newline at end of file diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java new file mode 100644 index 000000000..9f4ffb3da --- /dev/null +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/dynamo/DynamoJobRepositoryTest.java @@ -0,0 +1,479 @@ +package de.otto.edison.jobs.repository.dynamo; + +import de.otto.edison.jobs.domain.JobInfo; +import de.otto.edison.jobs.domain.JobInfo.JobStatus; +import de.otto.edison.jobs.domain.JobMessage; +import de.otto.edison.jobs.domain.Level; +import org.hamcrest.Matchers; +import org.hamcrest.collection.IsCollectionWithSize; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; + +import java.net.URI; +import java.time.Clock; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Optional; + +import static de.otto.edison.jobs.domain.JobInfo.JobStatus.ERROR; +import static de.otto.edison.jobs.domain.JobInfo.JobStatus.OK; +import static de.otto.edison.jobs.domain.JobInfo.builder; +import static de.otto.edison.jobs.domain.JobInfo.newJobInfo; +import static de.otto.edison.jobs.domain.JobMessage.jobMessage; +import static de.otto.edison.jobs.repository.dynamo.JobStructure.ID; +import static de.otto.edison.testsupport.matcher.OptionalMatchers.isAbsent; +import static de.otto.edison.testsupport.matcher.OptionalMatchers.isPresent; +import static java.time.Clock.fixed; +import static java.time.Clock.systemDefaultZone; +import static java.time.OffsetDateTime.now; +import static java.time.ZoneId.systemDefault; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.UUID.randomUUID; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static software.amazon.awssdk.services.dynamodb.model.ScalarAttributeType.S; + +@Testcontainers +class DynamoJobRepositoryTest { + + private static final String JOBS_TABLE_NAME = "FT6_DynamoDB_Jobs"; + + private static DynamoJobRepository testee; + + @Container + private static GenericContainer dynamodb = createTestContainer() + .withExposedPorts(8000); + + @AfterEach + void tearDown() { + deleteJobInfoTable(); + } + + @BeforeEach + void setUp() { + createJobInfoTable(); + testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); + } + + public static GenericContainer createTestContainer() { + return new GenericContainer<>("amazon/dynamodb-local:latest"); + } + + private static DynamoDbClient getDynamoDbClient() { + String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + + dynamodb.getMappedPort(8000); + + return DynamoDbClient.builder() + .endpointOverride(URI.create(endpointUri)) + .region(Region.EU_CENTRAL_1) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create("acc", "sec"))).build(); + } + + private Clock clock = systemDefaultZone(); + + @Test + void shouldCreateOrUpdateJob() { + //Given + final JobInfo savedjobInfo = jobInfo("http://localhost/foo", "T_FOO"); + testee.createOrUpdate(savedjobInfo); + + //When + testee.createOrUpdate(savedjobInfo); + + final Optional readJobInfo = testee.findOne(savedjobInfo.getJobId()); + + //Then + assertThat(readJobInfo.get(), equalTo(savedjobInfo)); + } + + @Test + void shouldFindJobInfoByUri() { + // given + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); + + // when + JobInfo job = newJobInfo(randomUUID().toString(), "MYJOB", clock, "localhost"); + repository.createOrUpdate(job); + + // then + assertThat(repository.findOne(job.getJobId()), isPresent()); + } + + @Test + void shouldReturnAbsentStatus() { + DynamoJobRepository repository = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 10); + assertThat(repository.findOne("some-nonexisting-job-id"), isAbsent()); + } + + @Test + void shouldNotFailToRemoveMissingJob() { + // when + testee.removeIfStopped("foo"); + // then + // no Exception is thrown... + } + + @Test + void shouldNotRemoveRunningJobs() { + // given + final String testUri = "test"; + testee.createOrUpdate(newJobInfo(testUri, "FOO", systemDefaultZone(), "localhost")); + // when + testee.removeIfStopped(testUri); + // then + assertThat(testee.size(), is(1L)); + + } + + @Test + void shouldRemoveJob() { + JobInfo stoppedJob = builder() + .setJobId("some/job/stopped") + .setJobType("test") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build(); + testee.createOrUpdate(stoppedJob); + testee.createOrUpdate(stoppedJob); + + testee.removeIfStopped(stoppedJob.getJobId()); + + assertThat(testee.size(), is(0L)); + } + + @Test + void shouldFindAll() { + // given + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final List jobInfos = testee.findAll(); + // then + assertThat(jobInfos.size(), is(2)); + assertThat(jobInfos.get(0).getJobId(), is("youngest")); + assertThat(jobInfos.get(1).getJobId(), is("oldest")); + } + + @Test + void shouldFindAllWithPaging() { + // given + testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 2); + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest1", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest2", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest3", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final List jobInfos = testee.findAll(); + // then + assertThat(jobInfos.size(), is(5)); + } + + @Test + void shouldFindAllinSizeOperation() { + // given + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final long count = testee.size(); + // then + assertThat(count, is(2L)); + } + + @Test + void shouldFindAllinSizeOperationWithPageing() { + // given + testee = new DynamoJobRepository(getDynamoDbClient(), JOBS_TABLE_NAME, 2); + testee.createOrUpdate(newJobInfo("oldest", "FOO", fixed(Instant.now().minusSeconds(1), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youn44444556gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("yo121ungest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youn333333gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youn1212gest", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + // when + final long count = testee.size(); + // then + assertThat(count, is(6L)); + } + + + @Test + void shouldFindLatestDistinct() { + // Given + Instant now = Instant.now(); + final JobInfo eins = newJobInfo("eins", "eins", fixed(now.plusSeconds(10), systemDefault()), "localhost"); + final JobInfo zwei = newJobInfo("zwei", "eins", fixed(now.plusSeconds(20), systemDefault()), "localhost"); + final JobInfo drei = newJobInfo("drei", "zwei", fixed(now.plusSeconds(30), systemDefault()), "localhost"); + final JobInfo vier = newJobInfo("vier", "drei", fixed(now.plusSeconds(40), systemDefault()), "localhost"); + final JobInfo fuenf = newJobInfo("fuenf", "drei", fixed(now.plusSeconds(50), systemDefault()), "localhost"); + + testee.createOrUpdate(eins); + testee.createOrUpdate(zwei); + testee.createOrUpdate(drei); + testee.createOrUpdate(vier); + testee.createOrUpdate(fuenf); + + // When + List latestDistinct = testee.findLatestJobsDistinct(); + + // Then + assertThat(latestDistinct, hasSize(3)); + assertThat(latestDistinct, Matchers.containsInAnyOrder(fuenf, zwei, drei)); + } + + + @Test + void shouldFindRunningJobsWithoutUpdatedSinceSpecificDate() { + // given + testee.createOrUpdate(newJobInfo("deadJob", "FOO", fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("running", "FOO", fixed(Instant.now(), systemDefault()), "localhost")); + + // when + final List jobInfos = testee.findRunningWithoutUpdateSince(now().minus(5, ChronoUnit.SECONDS)); + + // then + assertThat(jobInfos, IsCollectionWithSize.hasSize(1)); + assertThat(jobInfos.get(0).getJobId(), is("deadJob")); + } + + @Test + void shouldFindLatestByType() { + // given + final String type = "TEST"; + final String otherType = "OTHERTEST"; + + + testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); + + // when + final List jobInfos = testee.findLatestBy(type, 2); + + // then + assertThat(jobInfos.get(0).getJobId(), is("youngest")); + assertThat(jobInfos.get(1).getJobId(), is("oldest")); + assertThat(jobInfos, hasSize(2)); + } + + @Test + void shouldFindLatest() { + // given + final String type = "TEST"; + final String otherType = "OTHERTEST"; + testee.createOrUpdate(newJobInfo("oldest", type, fixed(Instant.now().minusSeconds(10), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("other", otherType, fixed(Instant.now().minusSeconds(5), systemDefault()), "localhost")); + testee.createOrUpdate(newJobInfo("youngest", type, fixed(Instant.now(), systemDefault()), "localhost")); + + // when + final List jobInfos = testee.findLatest(2); + + // then + assertThat(jobInfos.get(0).getJobId(), is("youngest")); + assertThat(jobInfos.get(1).getJobId(), is("other")); + assertThat(jobInfos, hasSize(2)); + } + + @Test + void shouldFindAllJobsOfSpecificType() { + // Given + final String type = "TEST"; + final String otherType = "OTHERTEST"; + testee.createOrUpdate(builder() + .setJobId("1") + .setJobType(type) + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build()); + testee.createOrUpdate(newJobInfo("2", otherType, systemDefaultZone(), "localhost")); + testee.createOrUpdate(newJobInfo("3", type, systemDefaultZone(), "localhost")); + + // When + final List jobsType1 = testee.findByType(type); + final List jobsType2 = testee.findByType(otherType); + + // Then + assertThat(jobsType1.size(), is(2)); + assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("1")), is(true)); + assertThat(jobsType1.stream().anyMatch(job -> job.getJobId().equals("3")), is(true)); + assertThat(jobsType2.size(), is(1)); + assertThat(jobsType2.stream().anyMatch(job -> job.getJobId().equals("2")), is(true)); + } + + @Test + void shouldFindStatusOfJob() { + //Given + final String type = "TEST"; + JobInfo jobInfo = newJobInfo("1", type, systemDefaultZone(), "localhost"); + testee.createOrUpdate(jobInfo); + + //When + JobStatus status = testee.findStatus("1"); + + //Then + assertThat(status, is(JobStatus.OK)); + } + + @Test + void shouldAppendMessageToJobInfo() { + + String someUri = "someUri"; + OffsetDateTime now = now(); + + //Given + JobInfo jobInfo = newJobInfo(someUri, "TEST", systemDefaultZone(), "localhost"); + testee.createOrUpdate(jobInfo); + + //When + JobMessage igelMessage = JobMessage.jobMessage(Level.WARNING, "Der Igel ist froh.", now); + testee.appendMessage(someUri, igelMessage); + + //Then + JobInfo jobInfoFromRepo = testee.findOne(someUri).get(); + + assertThat(jobInfoFromRepo.getMessages().size(), is(1)); + assertThat(jobInfoFromRepo.getMessages().get(0), is(igelMessage)); + assertThat(jobInfoFromRepo.getLastUpdated(), is(now.truncatedTo(ChronoUnit.MILLIS))); + } + + @Test + void shouldUpdateJobStatus() { + //Given + final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); //default jobStatus is 'OK' + testee.createOrUpdate(foo); + + //When + testee.setJobStatus(foo.getJobId(), ERROR); + JobStatus status = testee.findStatus("http://localhost/foo"); + + //Then + assertThat(status, is(ERROR)); + } + + @Test + void shouldUpdateJobLastUpdateTime() { + //Given + final JobInfo foo = jobInfo("http://localhost/foo", "T_FOO"); + testee.createOrUpdate(foo); + + OffsetDateTime myTestTime = OffsetDateTime.of(1979, 2, 5, 1, 2, 3, 1_000_000, ZoneOffset.UTC); + + //When + testee.setLastUpdate(foo.getJobId(), myTestTime); + + final Optional jobInfo = testee.findOne(foo.getJobId()); + + //Then + assertThat(jobInfo.get().getLastUpdated(), is(myTestTime)); + } + + @Test + void shouldClearJobInfos() { + //Given + // 25 is the max delete batch size + for (int i = 0; i < 26; i++) { + JobInfo stoppedJob = builder() + .setJobId("some/job/stopped" + i) + .setJobType("test") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .build(); + testee.createOrUpdate(stoppedJob); + } + //When + testee.deleteAll(); + + //Then + assertThat(testee.findAll(), is(emptyList())); + } + + @Test + public void shouldStoreAndRetrieveAllJobInfoWithoutMessages() { + // given + JobInfo job1 = builder() + .setJobId("someJobId1") + .setJobType("someJobType1") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setStopped(now(fixed(Instant.now().minusSeconds(7), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.OK) + .setLastUpdated(OffsetDateTime.now()) + .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage1", OffsetDateTime.now())) + .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage1", OffsetDateTime.now().plusSeconds(5L))) + .build(); + JobInfo job2 = builder() + .setJobId("someJobId2") + .setJobType("someJobType2") + .setStarted(now(fixed(Instant.now().minusSeconds(10), systemDefault()))) + .setHostname("localhost") + .setStatus(JobStatus.DEAD) + .setLastUpdated(OffsetDateTime.now()) + .addMessage(JobMessage.jobMessage(Level.INFO, "someInfoMessage2", OffsetDateTime.now())) + .addMessage(JobMessage.jobMessage(Level.ERROR, "someErrorMessage2", OffsetDateTime.now().plusSeconds(5L))) + .build(); + testee.createOrUpdate(job1); + testee.createOrUpdate(job2); + + // when + final List jobInfos = testee.findAllJobInfoWithoutMessages(); + // then + assertThat(jobInfos, hasSize(2)); + assertThat(jobInfos.get(0), is(job2.copy().setMessages(emptyList()).build())); + assertThat(jobInfos.get(1), is(job1.copy().setMessages(emptyList()).build())); + } + + private JobInfo jobInfo(final String jobId, final String type) { + return JobInfo.newJobInfo( + jobId, + type, + now(), now(), Optional.of(now()), OK, + asList( + jobMessage(Level.INFO, "foo", now()), + jobMessage(Level.WARNING, "bar", now())), + systemDefaultZone(), + "localhost" + ); + } + + private void createJobInfoTable() { + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(JOBS_TABLE_NAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName(ID.key()) + .attributeType(S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName(ID.key()) + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(1L) + .writeCapacityUnits(1L) + .build()) + .build()); + } + + private void deleteJobInfoTable() { + getDynamoDbClient().deleteTable(DeleteTableRequest.builder().tableName(JOBS_TABLE_NAME).build()); + } +} diff --git a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java index 93a69f5e8..2d3bbf57c 100644 --- a/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java +++ b/edison-jobs/src/test/java/de/otto/edison/jobs/repository/mongo/JobMetaRepositoryTest.java @@ -2,16 +2,29 @@ import de.otto.edison.jobs.domain.JobMeta; import de.otto.edison.jobs.repository.JobMetaRepository; +import de.otto.edison.jobs.repository.dynamo.DynamoJobMetaRepository; import de.otto.edison.jobs.repository.inmem.InMemJobMetaRepository; import de.otto.edison.mongo.configuration.MongoProperties; import de.otto.edison.testsupport.mongo.EmbeddedMongoHelper; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.dynamodb.model.*; import java.io.IOException; +import java.net.URI; import java.util.Collection; +import java.util.Set; import java.util.UUID; import static java.util.Arrays.asList; @@ -21,36 +34,105 @@ import static org.hamcrest.Matchers.*; import static org.hamcrest.core.Is.is; +@Testcontainers public class JobMetaRepositoryTest { + private static final String DYNAMO_JOB_META_TABLE_NAME = "FT6_DynamoDB_JobMeta"; + private static DynamoJobMetaRepository dynamoTestee = null; + @AfterAll public static void teardownMongo() { EmbeddedMongoHelper.stopMongoDB(); } @BeforeAll - public static void startMongo() throws IOException { + public static void initDbs() throws IOException { EmbeddedMongoHelper.startMongoDB(); + createDynamoTable(); + dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); + } + + @Container + private static GenericContainer dynamodb = createTestContainer() + .withExposedPorts(8000);; + + @BeforeEach + void setUp() { + createDynamoTable(); + dynamoTestee = new DynamoJobMetaRepository(getDynamoDbClient(), DYNAMO_JOB_META_TABLE_NAME); + } + + @AfterEach + public void tearDown() { + deleteDynamoTable(); } - public static Collection data() { + private static GenericContainer createTestContainer() { + return new GenericContainer<>("amazon/dynamodb-local:latest"); + } + + private static void createDynamoTable() { + try { + getDynamoDbClient().describeTable(DescribeTableRequest.builder() + .tableName(DYNAMO_JOB_META_TABLE_NAME) + .build()); + } catch (ResourceNotFoundException e) { + getDynamoDbClient().createTable(CreateTableRequest.builder() + .tableName(DYNAMO_JOB_META_TABLE_NAME) + .attributeDefinitions(AttributeDefinition.builder() + .attributeName("jobType") + .attributeType(ScalarAttributeType.S) + .build()) + .keySchema(KeySchemaElement.builder() + .attributeName("jobType") + .keyType(KeyType.HASH) + .build()) + .provisionedThroughput(ProvisionedThroughput.builder() + .readCapacityUnits(10L) + .writeCapacityUnits(10L) + .build()) + .build()); + } + } + + private static void deleteDynamoTable() { + DeleteTableRequest deleteTableRequest = DeleteTableRequest.builder() + .tableName(DYNAMO_JOB_META_TABLE_NAME).build(); + getDynamoDbClient().deleteTable(deleteTableRequest); + } + + private static Collection data() { return asList( new MongoJobMetaRepository(EmbeddedMongoHelper.getMongoClient().getDatabase("jobmeta-" + UUID.randomUUID()), "jobmeta", new MongoProperties()), - new InMemJobMetaRepository() + new InMemJobMetaRepository(), + dynamoTestee ); } + private static DynamoDbClient getDynamoDbClient() { + String endpointUri = "http://" + dynamodb.getContainerIpAddress() + ":" + + dynamodb.getMappedPort(8000); + return DynamoDbClient.builder() + .endpointOverride(URI.create(endpointUri)) + .region(Region.EU_CENTRAL_1) + .credentialsProvider(StaticCredentialsProvider + .create(AwsBasicCredentials.create("acc", "sec"))).build(); + } + @ParameterizedTest @MethodSource("data") public void shouldStoreAndGetValue(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when testee.setValue("someJob", "someKey", "someValue"); testee.setValue("someJob", "someOtherKey", "someDifferentValue"); - testee.setValue("someOtherJob", "someKey", "someOtherValue"); + //then assertThat(testee.getValue("someJob", "someKey"), is("someValue")); assertThat(testee.getValue("someJob", "someOtherKey"), is("someDifferentValue")); assertThat(testee.getValue("someOtherJob", "someKey"), is("someOtherValue")); @@ -59,9 +141,13 @@ public void shouldStoreAndGetValue(final JobMetaRepository testee) { @ParameterizedTest @MethodSource("data") public void shouldGetEmptyJobMeta(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(emptyMap())); assertThat(jobMeta.isDisabled(), is(false)); assertThat(jobMeta.getDisabledComment(), is("")); @@ -69,14 +155,17 @@ public void shouldGetEmptyJobMeta(final JobMetaRepository testee) { assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldGetJobMetaForRunningJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setRunningJob("someJob", "someId"); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(emptyMap())); assertThat(jobMeta.isDisabled(), is(false)); assertThat(jobMeta.getDisabledComment(), is("")); @@ -84,14 +173,17 @@ public void shouldGetJobMetaForRunningJob(final JobMetaRepository testee) { assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldGetJobMetaForDisabledJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.disable("someJob", "some comment"); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(emptyMap())); assertThat(jobMeta.isDisabled(), is(true)); assertThat(jobMeta.getDisabledComment(), is("some comment")); @@ -99,15 +191,18 @@ public void shouldGetJobMetaForDisabledJob(final JobMetaRepository testee) { assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldGetJobMetaForDisabledJobWithProperties(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.disable("someJob", "some comment"); testee.setValue("someJob", "someKey", "some value"); + + //when final JobMeta jobMeta = testee.getJobMeta("someJob"); + //then assertThat(jobMeta.getAll(), is(singletonMap("someKey", "some value"))); assertThat(jobMeta.isDisabled(), is(true)); assertThat(jobMeta.getDisabledComment(), is("some comment")); @@ -115,35 +210,43 @@ public void shouldGetJobMetaForDisabledJobWithProperties(final JobMetaRepository assertThat(jobMeta.getJobType(), is("someJob")); } - @ParameterizedTest @MethodSource("data") public void shouldEnableJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "_e_disabled", "foo"); + //when testee.enable("someJob"); + //then assertThat(testee.getValue("someJob", "_e_disabled"), is(nullValue())); } - @ParameterizedTest @MethodSource("data") public void shouldDisableJob(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when testee.disable("someJob", "some comment"); + //then assertThat(testee.getValue("someJob", "_e_disabled"), is("some comment")); } - @ParameterizedTest @MethodSource("data") public void shouldSetRunningJob(final JobMetaRepository testee) { + //given testee.deleteAll(); + + //when testee.setRunningJob("someJob", "someId"); + //then assertThat(testee.getRunningJob("someJob"), is("someId")); assertThat(testee.getValue("someJob", "_e_running"), is("someId")); } @@ -151,59 +254,68 @@ public void shouldSetRunningJob(final JobMetaRepository testee) { @ParameterizedTest @MethodSource("data") public void shouldDeleteAll(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.enable("foo"); testee.enable("bar"); + //when testee.deleteAll(); + //then assertThat(testee.findAllJobTypes(), is(empty())); } - @ParameterizedTest @MethodSource("data") public void shouldClearRunningJob(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "_e_running", "someId"); + //when testee.clearRunningJob("someJob"); + //then assertThat(testee.getRunningJob("someJob"), is(nullValue())); assertThat(testee.getValue("someJob", "_e_runnin"), is(nullValue())); } - @ParameterizedTest @MethodSource("data") public void shouldReturnNullForMissingKeys(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "someKey", "someValue"); + //when/then assertThat(testee.getValue("someJob", "someMissingKey"), is(nullValue())); assertThat(testee.getValue("someMissingJob", "someMissingKey"), is(nullValue())); } - @ParameterizedTest @MethodSource("data") public void shouldFindJobTypes(final JobMetaRepository testee) { + //given testee.deleteAll(); testee.setValue("someJob", "someKey", "someValue"); testee.setValue("someOtherJob", "someKey", "someOtherValue"); - assertThat(testee.findAllJobTypes(), containsInAnyOrder("someJob", "someOtherJob")); + //when + final Set allJobTypes = testee.findAllJobTypes(); + + //then + assertThat(allJobTypes, containsInAnyOrder("someJob", "someOtherJob")); } - @ParameterizedTest @MethodSource("data") public void shouldNotCreateIfExists(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.setValue("someJob", "someKey", "initialValue"); - // when + //when final boolean value = testee.createValue("someJob", "someKey", "newValue"); //then @@ -211,12 +323,13 @@ public void shouldNotCreateIfExists(final JobMetaRepository testee) { assertThat(testee.getValue("someJob", "someKey"), is("initialValue")); } - @ParameterizedTest @MethodSource("data") public void shouldCreateIfNotExists(final JobMetaRepository testee) { + //given testee.deleteAll(); - // when + + //when final boolean value = testee.createValue("someJob", "someKey", "someValue"); //then @@ -224,15 +337,14 @@ public void shouldCreateIfNotExists(final JobMetaRepository testee) { assertThat(testee.getValue("someJob", "someKey"), is("someValue")); } - @ParameterizedTest @MethodSource("data") public void shouldCreateTwoValuesWithoutException(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.createValue("someJob", "someKey", "someValue"); - // when + //when final boolean value = testee.createValue("someJob", "someOtherKey", "someOtherValue"); //then @@ -241,15 +353,14 @@ public void shouldCreateTwoValuesWithoutException(final JobMetaRepository testee assertThat(testee.getValue("someJob", "someOtherKey"), is("someOtherValue")); } - @ParameterizedTest @MethodSource("data") public void shouldReturnFalseIfCreateWasCalledTwice(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.createValue("someJob", "someKey", "someInitialValue"); - // when + //when final boolean value = testee.createValue("someJob", "someKey", "someValue"); //then @@ -257,15 +368,14 @@ public void shouldReturnFalseIfCreateWasCalledTwice(final JobMetaRepository test assertThat(testee.getValue("someJob", "someKey"), is("someInitialValue")); } - @ParameterizedTest @MethodSource("data") public void shouldNotKillOldFieldsOnCreate(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.setValue("someJob", "someKey", "someInitialValue"); - // when + //when final boolean value = testee.createValue("someJob", "someAtomicKey", "someValue"); //then @@ -274,18 +384,17 @@ public void shouldNotKillOldFieldsOnCreate(final JobMetaRepository testee) { assertThat(testee.getValue("someJob", "someAtomicKey"), is("someValue")); } - @ParameterizedTest @MethodSource("data") public void shouldUnsetKeyOnSetNullValue(final JobMetaRepository testee) { + //given testee.deleteAll(); - // given testee.setValue("someJob", "someKey", "someValue"); - // when + //when testee.setValue("someJob", "someKey", null); - // then + //then assertThat(testee.findAllJobTypes(), contains("someJob")); assertThat(testee.getValue("someJob", "someKey"), is(nullValue())); } diff --git a/edison-jobs/src/test/resources/application.yml b/edison-jobs/src/test/resources/application.yml index 7feb22365..c48af24f1 100644 --- a/edison-jobs/src/test/resources/application.yml +++ b/edison-jobs/src/test/resources/application.yml @@ -3,8 +3,16 @@ spring: name: testjobs server: - context-path: /testjobs + servlet: + context-path: /testjobs port: 8086 management: - context-path: /internal + endpoints: + web: + base-path: /internal + exposure: + include: '*' + endpoint: + loggers: + enabled: true diff --git a/edison-oauth/README.md b/edison-oauth/README.md index 6b65a6bd0..cfdc9b6eb 100644 --- a/edison-oauth/README.md +++ b/edison-oauth/README.md @@ -67,7 +67,7 @@ Afterwards, you can use an annotation at the controller method that checks the r for a certain `scope` inside the JWT Data: ```java - @RequestMapping(method = GET, value = "/secured/path", produces = MediaType.APPLICATION_JSON_VALUE) + @RequestMapping(method = GET, value = "/secured/path", produces = MediaType.APPLICATION_JSON) @ResponseBody @PreAuthorize("#oauth2.hasScope('some.oauth.scope')") public List getObjects() { diff --git a/edison-testsupport/build.gradle b/edison-testsupport/build.gradle index 3230096f2..83a474080 100644 --- a/edison-testsupport/build.gradle +++ b/edison-testsupport/build.gradle @@ -1,9 +1,11 @@ dependencies { - compileOnly libraries.togglz_console + compile libraries.togglz_console compileOnly libraries.togglz_spring_web compileOnly libraries.togglz_spring_boot_starter compileOnly libraries.aws_sdk_ssm compileOnly libraries.aws_sdk_s3 + compileOnly libraries.jackson_databind + compileOnly libraries.jackson_annotations compileOnly libraries.mongodb_driver compileOnly test_libraries.togglz_testing compileOnly test_libraries.embedded_mongo diff --git a/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java b/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java index 50b7c77bc..7861a36ac 100644 --- a/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java +++ b/edison-testsupport/src/main/java/de/otto/edison/testsupport/togglz/FeatureManagerSupport.java @@ -2,8 +2,8 @@ import org.togglz.core.Feature; import org.togglz.core.context.FeatureContext; +import org.togglz.core.manager.FeatureManager; import org.togglz.core.repository.FeatureState; -import org.togglz.core.util.FeatureAnnotations; import static org.togglz.core.context.FeatureContext.clearCache; import static org.togglz.core.context.FeatureContext.getFeatureManager; @@ -12,36 +12,46 @@ public class FeatureManagerSupport { public static void allEnabledFeatureConfig(final Class featureClass) { TestFeatureManager featureManager = new TestFeatureManager(featureClass); - enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureClass,featureManager); TestFeatureManagerProvider.setFeatureManager(featureManager); + allEnabledFeatureConfig(featureManager); + } + + public static void allEnabledFeatureConfig(FeatureManager featureManager) { + enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(featureManager); clearCache(); } public static void allDisabledFeatureConfig(final Class featureClass) { TestFeatureManager featureManager = new TestFeatureManager(featureClass); - for (Feature feature : featureClass.getEnumConstants()) { - featureManager.disable(feature); - } TestFeatureManagerProvider.setFeatureManager(featureManager); - clearCache(); + allDisabledFeatureConfig(featureManager); } + public static void allDisabledFeatureConfig(FeatureManager featureManager) { + featureManager.getFeatures().forEach(feature -> { + featureManager.setFeatureState(new FeatureState(feature, false)); + }); + clearCache(); + } public static void disable(final Feature feature) { FeatureContext.getFeatureManager().setFeatureState(new FeatureState(feature, false)); } - private static void enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(final Class featureClass, final TestFeatureManager featureManager) { - for (Feature feature : featureClass.getEnumConstants()) { - if (shouldRunInTests(feature)) { - featureManager.enable(feature); + private static void enableAllFeaturesThatAreOkToEnableByDefaultInAllTests(final FeatureManager featureManager) { + featureManager.getFeatures().forEach(feature -> { + if (shouldRunInTests(featureManager, feature)) { + featureManager.setFeatureState(new FeatureState(feature, true)); } - } + }); } public static boolean shouldRunInTests(Feature feature) { - String label = FeatureAnnotations.getLabel(feature); - return !label.contains("[inactiveInTests]"); + return shouldRunInTests(getFeatureManager(), feature); + } + + private static boolean shouldRunInTests(FeatureManager featureManager, Feature feature) { + return !featureManager.getMetaData(feature).getLabel().contains("[inactiveInTests]"); } public static void enable(final Feature feature) { diff --git a/edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java b/edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java new file mode 100644 index 000000000..d6708e911 --- /dev/null +++ b/edison-testsupport/src/test/java/de/otto/edison/testsupport/togglz/FeatureManagerSupportTest.java @@ -0,0 +1,43 @@ +package de.otto.edison.testsupport.togglz; + +import org.junit.jupiter.api.Test; +import org.togglz.core.Feature; +import org.togglz.core.annotation.Label; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.togglz.core.context.FeatureContext.getFeatureManager; + +class FeatureManagerSupportTest { + + @Test + void shouldEnableAllFeaturesThatAreNotInactiveInTests() { + //given + + //when + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + + //then + assertThat(getFeatureManager().isActive(TestFeatures.INACTIVE), is(false)); + assertThat(getFeatureManager().isActive(TestFeatures.ACTIVE), is(true)); + } + + @Test + void shouldDisableAllFeatures() { + //given + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + + //when + FeatureManagerSupport.allDisabledFeatureConfig(TestFeatures.class); + + //then + assertThat(getFeatureManager().isActive(TestFeatures.INACTIVE), is(false)); + assertThat(getFeatureManager().isActive(TestFeatures.ACTIVE), is(false)); + } + + public enum TestFeatures implements Feature { + @Label("should be inactive [inactiveInTests]") + INACTIVE, + ACTIVE, + } +} \ No newline at end of file diff --git a/edison-togglz/build.gradle b/edison-togglz/build.gradle index 42a30ffea..c9272219d 100644 --- a/edison-togglz/build.gradle +++ b/edison-togglz/build.gradle @@ -1,4 +1,7 @@ apply plugin: 'java-library' +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.3.50' +} dependencies { implementation project(":edison-core") @@ -14,6 +17,7 @@ dependencies { testImplementation project(":edison-testsupport") testImplementation project(":edison-mongo") + testImplementation test_libraries.togglz_testing testImplementation libraries.aws_sdk_s3 testImplementation test_libraries.embedded_mongo @@ -73,3 +77,6 @@ uploadArchives { } } } +repositories { + mavenCentral() +} diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java index 0a13a24bf..099d68150 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/configuration/TogglzConfiguration.java @@ -67,6 +67,7 @@ public TogglzConfig togglzConfig(final StateRepository stateRepository, } @Bean + @ConditionalOnMissingBean(FeatureManager.class) public FeatureManager featureManager(final TogglzConfig togglzConfig) throws Exception { final FeatureManagerFactory featureManagerFactory = new FeatureManagerFactory(); featureManagerFactory.setTogglzConfig(togglzConfig); diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java index 3ea5a1de4..76e79ce9b 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesController.java @@ -1,9 +1,9 @@ package de.otto.edison.togglz.controller; -import de.otto.edison.togglz.FeatureClassProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.togglz.core.manager.FeatureManager; import static de.otto.edison.togglz.controller.FeatureTogglesRepresentation.togglzRepresentation; import static org.springframework.web.bind.annotation.RequestMethod.GET; @@ -11,11 +11,11 @@ @RestController public class FeatureTogglesController { - private final FeatureClassProvider featureClassProvider; + private final FeatureManager featureManager; @Autowired - public FeatureTogglesController(final FeatureClassProvider featureClassProvider) { - this.featureClassProvider = featureClassProvider; + public FeatureTogglesController(final FeatureManager featureManager) { + this.featureManager = featureManager; } @RequestMapping( @@ -26,6 +26,6 @@ public FeatureTogglesController(final FeatureClassProvider featureClassProvider) method = GET ) public FeatureTogglesRepresentation getStatusAsJson() { - return togglzRepresentation(featureClassProvider); + return togglzRepresentation(featureManager); } } diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java index d6799b8e5..40ad55c56 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentation.java @@ -1,58 +1,41 @@ package de.otto.edison.togglz.controller; -import de.otto.edison.togglz.FeatureClassProvider; import net.jcip.annotations.Immutable; import org.togglz.core.Feature; -import org.togglz.core.annotation.Label; +import org.togglz.core.manager.FeatureManager; import java.util.Map; import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; -import static org.togglz.core.context.FeatureContext.getFeatureManager; @Immutable public class FeatureTogglesRepresentation { public final Map features; - private FeatureTogglesRepresentation(final Class featureClass) { - this.features = buildTogglzState(featureClass); + private FeatureTogglesRepresentation(final FeatureManager featureManager) { + this.features = buildTogglzState(featureManager); } - public static FeatureTogglesRepresentation togglzRepresentation(final FeatureClassProvider featureClassProvider) { - return new FeatureTogglesRepresentation(featureClassProvider.getFeatureClass()); + public static FeatureTogglesRepresentation togglzRepresentation(final FeatureManager featureManager) { + return new FeatureTogglesRepresentation(featureManager); } - private Map buildTogglzState(final Class featureClass) { - final Feature[] features = featureClass.getEnumConstants(); + private Map buildTogglzState(final FeatureManager featureManager) { + Feature[] features = featureManager.getFeatures().toArray(new Feature[]{}); return stream(features) .collect( - toMap(Feature::name, this::toFeatureToggleRepresentation) + toMap(Feature::name, feature -> toFeatureToggleRepresentation(feature, featureManager)) ); } - private FeatureToggleRepresentation toFeatureToggleRepresentation(final Feature feature) { - final Label label = getLabelAnnotation(feature); + private FeatureToggleRepresentation toFeatureToggleRepresentation(final Feature feature, FeatureManager featureManager) { + + final String label = featureManager.getMetaData(feature).getLabel(); return new FeatureToggleRepresentation( - label != null ? label.value() : feature.name(), - getFeatureManager().getFeatureState(feature).isEnabled(), + label != null ? label : feature.name(), + featureManager.getFeatureState(feature).isEnabled(), null); } - - public static Label getLabelAnnotation(Feature feature) { - try { - Class featureClass = feature.getClass(); - Label fieldAnnotation = featureClass.getField(feature.name()).getAnnotation(Label.class); - Label classAnnotation = featureClass.getAnnotation(Label.class); - - return fieldAnnotation != null ? fieldAnnotation : classAnnotation; - } catch (SecurityException e) { - // ignore - } catch (NoSuchFieldException e) { - // ignore - } - return null; - } - } diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java b/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java index d559a6f0a..a873ab74c 100644 --- a/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/s3/S3TogglzRepository.java @@ -9,8 +9,11 @@ import org.togglz.core.repository.StateRepository; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import static de.otto.edison.togglz.util.FeatureManagerSupport.getFeatureFromName; + /** * Togglz state repository, that fetches the s3 state async * to avoid s3 access while asking for togglz state. @@ -20,7 +23,7 @@ public class S3TogglzRepository implements StateRepository { private final static Logger LOG = LoggerFactory.getLogger(S3TogglzRepository.class); private static final int SCHEDULE_RATE_IN_MILLISECONDS = 60000; - private final Map cache = new ConcurrentHashMap<>(); + private final Map cache = new ConcurrentHashMap<>(); private final FeatureStateConverter featureStateConverter; public S3TogglzRepository(final FeatureStateConverter featureStateConverter) { @@ -30,7 +33,7 @@ public S3TogglzRepository(final FeatureStateConverter featureStateConverter) { @Override public FeatureState getFeatureState(final Feature feature) { - final CacheEntry cachedEntry = cache.get(feature); + final CacheEntry cachedEntry = cache.get(feature.name()); if (cachedEntry != null) { return cachedEntry.getState(); @@ -38,7 +41,7 @@ public FeatureState getFeatureState(final Feature feature) { // no cache hit - refresh state from delegate final FeatureState featureState = featureStateConverter.retrieveFeatureStateFromS3(feature); - cache.put(feature, new CacheEntry(featureState)); + cache.put(feature.name(), new CacheEntry(featureState)); return featureState; } @@ -46,7 +49,7 @@ public FeatureState getFeatureState(final Feature feature) { @Override public void setFeatureState(final FeatureState featureState) { featureStateConverter.persistFeatureStateToS3(featureState); - cache.put(featureState.getFeature(), new CacheEntry(featureState)); + cache.put(featureState.getFeature().name(), new CacheEntry(featureState)); } @Scheduled(initialDelay = 0, fixedRate = SCHEDULE_RATE_IN_MILLISECONDS) @@ -56,7 +59,14 @@ protected void prefetchFeatureStates() { initializeFeatureStates(); } else { LOG.debug("Refreshing state for features"); - cache.replaceAll((feature, cacheEntry) -> new CacheEntry(featureStateConverter.retrieveFeatureStateFromS3(feature))); + + cache.replaceAll((featureName, cacheEntry) -> { + Optional featureFromName = getFeatureFromName(featureName); + if (featureFromName.isPresent()) { + return new CacheEntry(featureStateConverter.retrieveFeatureStateFromS3(featureFromName.get())); + } + return null; + }); } } diff --git a/edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java b/edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java new file mode 100644 index 000000000..6f0f2cd6e --- /dev/null +++ b/edison-togglz/src/main/java/de/otto/edison/togglz/util/FeatureManagerSupport.java @@ -0,0 +1,14 @@ +package de.otto.edison.togglz.util; + +import org.togglz.core.Feature; + +import java.util.Optional; + +import static org.togglz.core.context.FeatureContext.getFeatureManager; + +public class FeatureManagerSupport { + + public static Optional getFeatureFromName(String name) { + return getFeatureManager().getFeatures().stream().filter(feature -> feature.name().equals(name)).findAny(); + } +} diff --git a/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java b/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java index 0c9778a65..a60233240 100644 --- a/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/acceptance/togglz/FeatureTogglesControllerAcceptanceTest.java @@ -2,19 +2,33 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import de.otto.edison.testsupport.togglz.FeatureManagerSupport; +import de.otto.edison.togglz.TestFeatures; import de.otto.edison.togglz.TestServer; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.web.client.RestTemplate; +import org.togglz.core.Feature; +import org.togglz.core.manager.FeatureManager; +import org.togglz.core.manager.TogglzConfig; +import org.togglz.core.repository.StateRepository; +import org.togglz.core.repository.mem.InMemoryStateRepository; +import org.togglz.core.user.NoOpUserProvider; +import org.togglz.core.user.UserProvider; import java.io.IOException; @@ -26,6 +40,7 @@ @ExtendWith(SpringExtension.class) @SpringBootTest(webEnvironment = RANDOM_PORT, classes = {TestServer.class}) +@ContextConfiguration(classes = {FeatureTogglesControllerAcceptanceTest.TogglzConfiguration.class}) @ActiveProfiles("test") public class FeatureTogglesControllerAcceptanceTest { @@ -34,11 +49,19 @@ public class FeatureTogglesControllerAcceptanceTest { @Autowired private ObjectMapper objectMapper; + @Autowired + private FeatureManager featureManager; + @LocalServerPort private int port; + @BeforeEach + void setUp() { + FeatureManagerSupport.allEnabledFeatureConfig(featureManager); + } + @Test - public void shouldTogglesAsJson() { + public void shouldReturnTogglesAsJson() { // when ResponseEntity resource = getResource("http://localhost:" + port + "/togglztest/internal/toggles"); @@ -68,5 +91,29 @@ JsonNode jsonNode(ResponseEntity resource) { } } + @Configuration + static class TogglzConfiguration{ + + @Bean + @Profile("test") + public TogglzConfig togglzConfig() { + return new TogglzConfig() { + @Override + public Class getFeatureClass() { + return TestFeatures.class; + } + @Override + public StateRepository getStateRepository() { + return new InMemoryStateRepository(); + } + + @Override + public UserProvider getUserProvider() { + return new NoOpUserProvider(); + } + }; + } + + } } diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java index 85c88326a..0a4915816 100644 --- a/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/DefaultTogglzConfigTest.java @@ -1,5 +1,7 @@ package de.otto.edison.togglz; +import de.otto.edison.testsupport.togglz.FeatureManagerSupport; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -22,6 +24,11 @@ public class DefaultTogglzConfigTest { @Autowired private TogglzConfig togglzConfig; + @BeforeEach + void setUp() { + FeatureManagerSupport.allEnabledFeatureConfig(TestFeatures.class); + } + @Test public void shouldCreateTogglzConfigBySpring() { assertThat(togglzConfig, is(not(nullValue()))); diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java index cbfc132f0..1f81b2921 100644 --- a/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/controller/FeatureTogglesRepresentationTest.java @@ -3,6 +3,8 @@ import de.otto.edison.togglz.EmptyFeatures; import de.otto.edison.togglz.TestFeatures; import org.junit.jupiter.api.Test; +import org.togglz.core.manager.FeatureManager; +import org.togglz.testing.TestFeatureManager; import java.util.Map; @@ -17,15 +19,17 @@ public class FeatureTogglesRepresentationTest { @Test public void testGetFeatureRepresentation() { - testee = togglzRepresentation(() -> TestFeatures.class); + FeatureManager featureManager = new TestFeatureManager(TestFeatures.class); + testee = togglzRepresentation(featureManager); final Map features = testee.features; - assertThat(features.get("TEST_FEATURE"), is(new FeatureToggleRepresentation("a test feature toggle", true, null))); + assertThat(features.get("TEST_FEATURE"), is(new FeatureToggleRepresentation("a test feature toggle", false, null))); } @Test public void testGetEmptyFeatureRepresentation() { - testee = togglzRepresentation(() -> EmptyFeatures.class); + FeatureManager featureManager = new TestFeatureManager(EmptyFeatures.class); + testee = togglzRepresentation(featureManager); final Map features = testee.features; assertThat(features, is(notNullValue())); diff --git a/edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java b/edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java new file mode 100644 index 000000000..178036275 --- /dev/null +++ b/edison-togglz/src/test/java/de/otto/edison/togglz/util/FeatureManagerSupportTest.java @@ -0,0 +1,32 @@ +package de.otto.edison.togglz.util; + +import de.otto.edison.togglz.TestFeatures; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.togglz.core.Feature; + +import java.util.Optional; + +import static de.otto.edison.testsupport.togglz.FeatureManagerSupport.allEnabledFeatureConfig; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; + +class FeatureManagerSupportTest { + + @BeforeEach + void setUp() { + allEnabledFeatureConfig(TestFeatures.class); + } + + @Test + void shouldReturnTheCorrectFeature() { + assertThat(FeatureManagerSupport.getFeatureFromName("TEST_FEATURE").get(), is(TestFeatures.TEST_FEATURE)); + } + + @Test + void shouldReturnTheAnEmptyFeatureIfNameisNotKnown() { + Optional unknwonFeature = FeatureManagerSupport.getFeatureFromName("UNKNWON_FEATURE"); + assertFalse(unknwonFeature.isPresent()); + } +} \ No newline at end of file diff --git a/edison-togglz/src/test/resources/application.yml b/edison-togglz/src/test/resources/application.yml index dd200d49b..732754d85 100644 --- a/edison-togglz/src/test/resources/application.yml +++ b/edison-togglz/src/test/resources/application.yml @@ -8,3 +8,12 @@ server: servlet: context-path: /togglztest port: 8085 + +management: + endpoints: + web: + exposure: + include: '*' + endpoint: + loggers: + enabled: true diff --git a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java index 90ec11531..35c587a48 100644 --- a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java +++ b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentation.java @@ -5,11 +5,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import de.otto.edison.hal.HalRepresentation; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL; import static de.otto.edison.hal.Link.profile; @@ -18,14 +14,12 @@ @JsonDeserialize(builder = ErrorHalRepresentation.Builder.class) public class ErrorHalRepresentation extends HalRepresentation { - private static final String PROFILE_ERROR = "http://spec.otto.de/profiles/error"; - private final String errorMessage; private final Map>> errors; private ErrorHalRepresentation(Builder builder) { super(linkingTo() - .array(profile(PROFILE_ERROR)) + .array(profile(builder.profile)) .build() ); this.errors = builder.errors; @@ -74,6 +68,7 @@ public String getErrorMessage() { public static final class Builder { private Map>> errors = new HashMap<>(); private String errorMessage; + private String profile = ""; private Builder() { } @@ -82,6 +77,11 @@ public ErrorHalRepresentation build() { return new ErrorHalRepresentation(this); } + public Builder withProfile(String profile) { + this.profile = profile; + return this; + } + public Builder withErrors(Map>> errors) { this.errors = errors; return this; diff --git a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java index 239a1cb8e..691d6b1b3 100644 --- a/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java +++ b/edison-validation/src/main/java/de/otto/edison/validation/web/ErrorHalRepresentationFactory.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.support.ResourceBundleMessageSource; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; @@ -17,15 +18,28 @@ public class ErrorHalRepresentationFactory { private final ResourceBundleMessageSource messageSource; private final ObjectMapper objectMapper; + private final String errorProfile; @Autowired - public ErrorHalRepresentationFactory(ResourceBundleMessageSource edisonValidationMessageSource, ObjectMapper objectMapper) { + public ErrorHalRepresentationFactory( + ResourceBundleMessageSource edisonValidationMessageSource, + ObjectMapper objectMapper, + @Value("${edison.validation.error-profile:http://spec.otto.de/profiles/error}") String errorProfile) { this.messageSource = edisonValidationMessageSource; this.objectMapper = objectMapper; + this.errorProfile = errorProfile; + } + + public ErrorHalRepresentation halRepresentationForErrorMessage(String errorMessage) { + return ErrorHalRepresentation.builder() + .withProfile(errorProfile) + .withErrorMessage(errorMessage) + .build(); } public ErrorHalRepresentation halRepresentationForValidationErrors(Errors validationResult) { ErrorHalRepresentation.Builder builder = ErrorHalRepresentation.builder() + .withProfile(errorProfile) .withErrorMessage(String.format("Validation failed. %d error(s)", validationResult.getErrorCount())); validationResult.getAllErrors() diff --git a/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java b/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java index dd2389570..d5fdc5c7b 100644 --- a/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java +++ b/edison-validation/src/main/java/de/otto/edison/validation/web/ValidationExceptionHandler.java @@ -15,7 +15,7 @@ public class ValidationExceptionHandler { private static final MediaType APPLICATION_HAL_JSON_ERROR = MediaType.parseMediaType("application/hal+json; " + - "profiles=\"http://spec.otto.de/profiles/error\"; charset=utf-8"); + "profiles=\"http://spec.otto.de/profiles/error\""); private final ErrorHalRepresentationFactory errorHalRepresentationFactory; @Autowired diff --git a/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java b/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java index dee78fa09..571542885 100644 --- a/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java +++ b/edison-validation/src/test/java/de/otto/edison/validation/web/ErrorHalRepresentationFactoryTest.java @@ -31,10 +31,27 @@ public void setUp() { } + @Test + public void shouldBuildRepresentationForErrorMessage() { + // given + String someErrorProfile = "someErrorProfile"; + String someErrorMessage = "someErrorMessage"; + final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper(), someErrorProfile); + + // when + ErrorHalRepresentation errorHalRepresentation = factory.halRepresentationForErrorMessage(someErrorMessage); + + // then + assertThat(errorHalRepresentation.getErrors().isEmpty(), is(true)); + assertThat(errorHalRepresentation.getErrorMessage(), is(someErrorMessage)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").isPresent(), is(true)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").get().getHref(), is(someErrorProfile)); + } + @Test public void shouldBuildRepresentationForValidationResults() { // given - final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper()); + final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper(), "someErrorProfile"); // when final Errors mockErrors = mock(Errors.class); @@ -51,6 +68,8 @@ public void shouldBuildRepresentationForValidationResults() { // then assertThat(errorHalRepresentation.getErrorMessage(), is("Validation failed. 1 error(s)")); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").isPresent(), is(true)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").get().getHref(), is("someErrorProfile")); final List> listOfViolations = errorHalRepresentation.getErrors().get("xyzField"); assertThat(listOfViolations, hasSize(1)); assertThat(listOfViolations.get(0), hasEntry("key", "text.not_empty")); @@ -61,7 +80,7 @@ public void shouldBuildRepresentationForValidationResults() { @Test public void shouldNotCrashOnNullValues() { // given - final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper()); + final ErrorHalRepresentationFactory factory = new ErrorHalRepresentationFactory(messageSource, new ObjectMapper(), "someErrorProfile"); // when final Errors mockErrors = mock(Errors.class); @@ -78,6 +97,8 @@ public void shouldNotCrashOnNullValues() { // then assertThat(errorHalRepresentation.getErrorMessage(), is("Validation failed. 1 error(s)")); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").isPresent(), is(true)); + assertThat(errorHalRepresentation.getLinks().getLinkBy("profile").get().getHref(), is("someErrorProfile")); final List> listOfViolations = errorHalRepresentation.getErrors().get("xyzField"); assertThat(listOfViolations, hasSize(1)); assertThat(listOfViolations.get(0), hasEntry("key", "text.not_empty")); diff --git a/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java b/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java index 367b429b5..b0ea273eb 100644 --- a/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java +++ b/edison-validation/src/test/java/de/otto/edison/validation/web/ValidationExceptionHandlerAcceptanceTest.java @@ -60,10 +60,10 @@ public void shouldValidateAndProduceErrorRepresentation() { .then() .assertThat() .statusCode(is(422)).and() - .header("Content-type", Matchers.containsString(";charset=utf-8")) - .content("errors.id[0].key", Collections.emptyList(), is("id.invalid")) - .content("errors.id[0].message", Collections.emptyList(), is("Ungueltiger Id-Wert.")) - .content("errors.id[0].rejected", Collections.emptyList(), is("_!NON_SAFE_ID!!?**")); + .header("Content-type", Matchers.containsString("application/hal+json")) + .body("errors.id[0].key", Collections.emptyList(), is("id.invalid")) + .body("errors.id[0].message", Collections.emptyList(), is("Ungueltiger Id-Wert.")) + .body("errors.id[0].rejected", Collections.emptyList(), is("_!NON_SAFE_ID!!?**")); } public static class TestConfiguration { diff --git a/examples/example-jobs/src/main/resources/application.yml b/examples/example-jobs/src/main/resources/application.yml index 39bf51613..aa8f57902 100644 --- a/examples/example-jobs/src/main/resources/application.yml +++ b/examples/example-jobs/src/main/resources/application.yml @@ -22,7 +22,11 @@ management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true edison: # disable graceful shutdown diff --git a/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java b/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java index e3abe710c..c17ab8021 100644 --- a/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java +++ b/examples/example-jobs/src/test/java/de/otto/edison/example/ExampleJobsSmokeTest.java @@ -43,21 +43,21 @@ public void shouldRenderMainPage() { public void shouldHaveStatusEndpoint() { final ResponseEntity response = this.restTemplate.getForEntity("/internal/status?format=json", String.class); assertThat(response.getStatusCodeValue()).isEqualTo(200); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getBody()).startsWith("{"); } @Test public void shouldHaveHealthCheck() { final ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isEqualTo(200); } @Test public void shouldHaveJobDefinitions() throws JSONException { final ResponseEntity response = this.restTemplate.getForEntity("/internal/jobdefinitions?format=json", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isEqualTo(200); JSONAssert.assertEquals("{\n" + " \"links\" : [ {\n" + @@ -91,7 +91,7 @@ public void shouldHaveJobDefinitions() throws JSONException { @Test public void shouldHaveFooJobDefinition() throws JSONException { final ResponseEntity response = this.restTemplate.getForEntity("/internal/jobdefinitions/foo?format=json", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isEqualTo(200); JSONAssert.assertEquals("{\n" + " \"type\" : \"Foo\",\n" + diff --git a/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java b/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java index 6c7dc9ad7..f3ef9e9d2 100644 --- a/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java +++ b/examples/example-oauth/src/main/java/de/otto/edison/example/ApiController.java @@ -1,5 +1,6 @@ package de.otto.edison.example; +import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.ResponseBody; @@ -12,7 +13,7 @@ public class ApiController { @RequestMapping( value = "/api/hello", - produces = "application/json", + produces = MediaType.APPLICATION_JSON_VALUE, method = GET) @ResponseBody @PreAuthorize("#oauth2.hasScope('hello.read')") diff --git a/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java b/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java index ae088e024..77a23d261 100644 --- a/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java +++ b/examples/example-oauth/src/test/java/de/otto/edison/example/ApiControllerIntegrationTest.java @@ -12,6 +12,7 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.junit.jupiter.SpringExtension; +import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.RANDOM_PORT; @@ -49,13 +50,13 @@ public void shouldReturnHelloResponseWithValidOauthToken() throws Exception { .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") .addHeader(AUTHORIZATION, bearerToken) - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(200)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), containsString(MediaType.APPLICATION_JSON_VALUE)); assertThat(response.getResponseBody(), is("{\"hello\": \"world\"}")); } @@ -69,13 +70,13 @@ public void shouldReturn403WhenRequestingWithInvalidScopeInOauthToken() throws E .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") .addHeader(AUTHORIZATION, bearerToken) - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(403)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_VALUE)); } @Test @@ -88,13 +89,13 @@ public void shouldReturn403WhenRequestingWithInvalidOauthToken() throws Exceptio .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") .addHeader(AUTHORIZATION, bearerToken) - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(403)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_VALUE)); } @Test @@ -105,13 +106,13 @@ public void shouldReturn403WhenRequestingWithoutOauthToken() throws Exception { final Response response = asyncHttpClient .prepareGet(baseUrl + "/api/hello") .addQueryParam("context", "mode") - .addHeader(ACCEPT, MediaType.APPLICATION_JSON_UTF8_VALUE) + .addHeader(ACCEPT, MediaType.APPLICATION_JSON_VALUE) .execute() .get(); // Then assertThat(response.getStatusCode(), is(403)); - assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_UTF8_VALUE)); + assertThat(response.getContentType(), is(MediaType.APPLICATION_JSON_VALUE)); } diff --git a/examples/example-status/src/main/resources/application.yml b/examples/example-status/src/main/resources/application.yml index da96fea24..cfe67fa89 100644 --- a/examples/example-status/src/main/resources/application.yml +++ b/examples/example-status/src/main/resources/application.yml @@ -15,7 +15,11 @@ management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true edison: gracefulshutdown: diff --git a/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java b/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java index 3a8b83d34..ec39efabe 100644 --- a/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java +++ b/examples/example-status/src/test/java/de/otto/edison/example/ExampleStatusSmokeTest.java @@ -32,14 +32,14 @@ public void shouldRenderMainPage() { public void shouldHaveStatusEndpoint() { final ResponseEntity response = this.restTemplate.getForEntity("/internal/status", String.class); assertThat(response.getStatusCodeValue()).isEqualTo(200); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getBody()).startsWith("{"); } @Test public void shouldHaveHealthCheck() { final ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isIn(200, 503); } diff --git a/examples/example-togglz/src/main/resources/application.yml b/examples/example-togglz/src/main/resources/application.yml index 7b863d211..b585f92ab 100644 --- a/examples/example-togglz/src/main/resources/application.yml +++ b/examples/example-togglz/src/main/resources/application.yml @@ -17,7 +17,11 @@ management: endpoints: web: base-path: /actuator - expose: '*' + exposure: + include: '*' + endpoint: + loggers: + enabled: true edison: gracefulshutdown: diff --git a/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java b/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java index 219e30b21..2d9ec53ab 100644 --- a/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java +++ b/examples/example-togglz/src/test/java/de/otto/edison/example/ExampleTogglzSmokeTest.java @@ -39,14 +39,14 @@ public void shouldRenderTogglzConsole() { public void shouldHaveStatusEndpoint() { final ResponseEntity response = this.restTemplate.getForEntity("/internal/status", String.class); assertThat(response.getStatusCodeValue()).isEqualTo(200); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getBody()).startsWith("{"); } @Test public void shouldHaveHealthCheck() { final ResponseEntity response = this.restTemplate.getForEntity("/actuator/health", String.class); - assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON_UTF8); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); assertThat(response.getStatusCodeValue()).isIn(200, 503); } diff --git a/examples/example-togglz/src/test/resources/application.yml b/examples/example-togglz/src/test/resources/application.yml index e03ff18f1..2753d2cf4 100644 --- a/examples/example-togglz/src/test/resources/application.yml +++ b/examples/example-togglz/src/test/resources/application.yml @@ -1,3 +1,12 @@ spring: main: - allow-bean-definition-overriding: true \ No newline at end of file + allow-bean-definition-overriding: true + +management: + endpoints: + web: + exposure: + include: '*' + endpoint: + loggers: + enabled: true \ No newline at end of file diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 5e75ac344..cdebd350c 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -4,42 +4,43 @@ */ ext { versions = [ - spring_boot : '2.1.6.RELEASE', - spring : '5.1.8.RELEASE', - spring_security_core : '5.1.5.RELEASE', - spring_security_web : '5.1.5.RELEASE', - spring_security_oauth : '2.3.6.RELEASE', - spring_security_jwt : '1.0.10.RELEASE', - async_http_client : '2.9.0', //Use 2.9.0 because next update updates netty to 4.1.36 which is incompatible with AWS SDK as of 2019-06-02 (needs 4.1.33) + spring_boot : '2.2.0.RELEASE', + spring : '5.2.0.RELEASE', + spring_security_core : '5.2.0.RELEASE', + spring_security_web : '5.2.0.RELEASE', + spring_security_oauth : '2.3.7.RELEASE', + spring_security_jwt : '1.0.11.RELEASE', + async_http_client : '2.10.4', jcip_annotations : '1.0', logback_classic : '1.2.3', javax_servlet_api : '3.1.0', togglz : '2.6.1.Final', - mongodb_driver : '3.8.1', - caffeine : '2.7.0', + mongodb_driver : '3.11.0', + caffeine : '2.8.0', json_path : '2.4.0', unboundid_ldapsdk_minimal_edition: '3.2.1', hibernate_validator : '6.0.17.Final', edison_hal : '2.0.2', validator_collection : '2.2.0', slf4j : '1.7.26', - aws_sdk : '2.6.4', + aws_sdk : '2.10.56', java_validation_api : '2.0.1.Final', - java_xml : '2.3.0' + java_xml : '2.3.0', + jackson : '2.10.0' ] test_versions = [ - junit : '5.2.0', - hamcrest : '1.3', + junit : '5.5.2', + hamcrest : '2.1', mockito_core : '2.28.2', jsonassert : '1.5.0', - rest_assured : '3.1.1', + rest_assured : '4.1.2', embedded_mongo: '2.2.0', - testcontainers: '1.8.3' + testcontainers: '1.12.3' ] plugin_versions = [ - versions : '0.15.0', + versions : '0.25.0', jacoco : '0.8.2', - nexus_staging: '0.11.0' + nexus_staging: '0.21.1' ] libraries = [ @@ -57,6 +58,7 @@ ext { spring_security_jwt : "org.springframework.security:spring-security-jwt:${versions.spring_security_jwt}", aws_sdk_s3 : "software.amazon.awssdk:s3:${versions.aws_sdk}", aws_sdk_ssm : "software.amazon.awssdk:ssm:${versions.aws_sdk}", + aws_sdk_dynamodb : "software.amazon.awssdk:dynamodb:${versions.aws_sdk}", async_http_client : "org.asynchttpclient:async-http-client:${versions.async_http_client}", jcip_annotations : "net.jcip:jcip-annotations:${versions.jcip_annotations}", logback_classic : "ch.qos.logback:logback-classic:${versions.logback_classic}", @@ -65,13 +67,17 @@ ext { togglz_console : "org.togglz:togglz-console:${versions.togglz}", togglz_spring_web : "org.togglz:togglz-spring-web:${versions.togglz}", togglz_spring_boot_starter : "org.togglz:togglz-spring-boot-starter:${versions.togglz}", + togglz_testing : "org.togglz:togglz-testing:${versions.togglz}", mongodb_driver : "org.mongodb:mongodb-driver:${versions.mongodb_driver}", caffeine : "com.github.ben-manes.caffeine:caffeine:${versions.caffeine}", unboundid_ldapsdk_minimal_edition : "com.unboundid:unboundid-ldapsdk-minimal-edition:${versions.unboundid_ldapsdk_minimal_edition}", hibernate_validator : "org.hibernate.validator:hibernate-validator:${versions.hibernate_validator}", java_validation_api : "javax.validation:validation-api:${versions.java_validation_api}", edison_hal : "de.otto.edison:edison-hal:${versions.edison_hal}", - validator_collection : "cz.jirutka.validator:validator-collection:${versions.validator_collection}" + validator_collection : "cz.jirutka.validator:validator-collection:${versions.validator_collection}", + jackson : "com.fasterxml.jackson.core:jackson-core:${versions.jackson}", + jackson_databind : "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}", + jackson_annotations : "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" ] java_xml = [ "javax.xml.bind:jaxb-api:${versions.java_xml}", @@ -92,7 +98,8 @@ ext { embedded_mongo : "de.flapdoodle.embed:de.flapdoodle.embed.mongo:${test_versions.embedded_mongo}", json_path : "com.jayway.jsonpath:json-path:${versions.json_path}", rest_assured : "io.rest-assured:rest-assured:${test_versions.rest_assured}", - testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}" +// testcontainers : "org.testcontainers:testcontainers:${test_versions.testcontainers}", + testcontainers : "org.testcontainers:junit-jupiter:${test_versions.testcontainers}" ] gradle_plugins = [ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 18d5b6e0e..39b9f2a6f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.5.1-bin.zip + +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists \ No newline at end of file diff --git a/gs b/gs deleted file mode 100644 index e69de29bb..000000000