Skip to content

Commit

Permalink
BFD-2682: migrator app progress tracking via SQS (#1829)
Browse files Browse the repository at this point in the history 
* Adds progress tracking via SQS.

* Refactors MigratorApp to make it easier to read and test.
  • Loading branch information
@brianburton
brianburton authored Jul 19, 2023
1 parent 4776a20 commit 75ba751
Show file tree
Hide file tree
Showing 18 changed files with 1,165 additions and 93 deletions.
34 changes: 34 additions & 0 deletions apps/bfd-db-migrator/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@
<artifactId>reflections</artifactId>
<version>${reflections.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<!-- Used to interact with AWS' SSM service -->
<groupId>software.amazon.awssdk</groupId>
<artifactId>sqs</artifactId>
</dependency>
<dependency>
<!-- An implementation of the Log4J API that routes everything through to SLF4J,
instead. -->
Expand Down Expand Up @@ -115,12 +125,36 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<!-- Adds convenience features for working with mocks in tests -->
<groupId>org.mockito</groupId>
<artifactId>mockito-junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>gov.cms.bfd</groupId>
<artifactId>bfd-shared-test-utils</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<!-- Used to write fluent assertions in unit tests. -->
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>localstack</artifactId>
<version>${testcontainers.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
92 changes: 78 additions & 14 deletions apps/bfd-db-migrator/src/main/java/gov/cms/bfd/migrator/app/AppConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,48 @@
import gov.cms.bfd.sharedutils.config.LayeredConfiguration;
import gov.cms.bfd.sharedutils.config.MetricOptions;
import gov.cms.bfd.sharedutils.database.DatabaseOptions;
import java.net.URI;
import java.util.Map;
import java.util.function.Function;
import javax.annotation.Nullable;
import lombok.Getter;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.sqs.SqsClient;

/** Models the configuration options for the application. */
public class AppConfiguration extends BaseAppConfiguration {
/** Name of setting containing name of SQS queue to which progress messages can be sent. */
public static final String ENV_VAR_KEY_SQS_QUEUE_NAME = "DB_MIGRATOR_SQS_QUEUE";
/** Name of setting containing alternative endpoint URL for SQS service. */
public static final String ENV_VAR_KEY_SQS_ENDPOINT = "DB_MIGRATOR_SQS_ENDPOINT";
/** Name of setting containing region name for SQS service queue. */
public static final String ENV_VAR_KEY_SQS_REGION = "DB_MIGRATOR_SQS_REGION";
/** Name of setting containing access key for SQS service. */
public static final String ENV_VAR_KEY_SQS_ACCESS_KEY = "DB_MIGRATOR_SQS_ACCESS_KEY";
/** Name of setting containing secret key for SQS service. */
public static final String ENV_VAR_KEY_SQS_SECRET_KEY = "DB_MIGRATOR_SQS_SECRET_KEY";

/**
* Controls where flyway looks for migration scripts. If not set (null or empty string) flyway
* will use it's default location {@code src/main/resources/db/migration}. This is primarily for
* the integration tests, so we can run test migrations under an arbitrary directory full of
* scripts.
*/
private final String flywayScriptLocationOverride;
@Getter private final String flywayScriptLocationOverride;

/**
* Used to communicate with SQS for sending progress messages. Null if SQS progress tracking has
* not been enabled.
*/
@Nullable @Getter private final SqsClient sqsClient;

/**
* Name of SQS queue to send progress messages to. Empty string if SQS progress tracking has not
* been enabled.
*/
@Getter private final String sqsQueueName;

/**
* Constructs a new {@link AppConfiguration} instance.
Expand All @@ -26,49 +56,83 @@ public class AppConfiguration extends BaseAppConfiguration {
* @param databaseOptions the value to use for {@link #getDatabaseOptions()}
* @param flywayScriptLocationOverride if non-empty, will override the default location that
* flyway looks for migration scripts
* @param sqsClient null or a valid {@link SqsClient}
* @param sqsQueueName name of queue to post progress to (empty if none)
*/
private AppConfiguration(
MetricOptions metricOptions,
DatabaseOptions databaseOptions,
String flywayScriptLocationOverride) {
String flywayScriptLocationOverride,
@Nullable SqsClient sqsClient,
String sqsQueueName) {
super(metricOptions, databaseOptions);
this.flywayScriptLocationOverride = flywayScriptLocationOverride;
}

/**
* Gets the flyway script location override.
*
* @return the flyway script location override
*/
public String getFlywayScriptLocationOverride() {
return flywayScriptLocationOverride;
this.sqsClient = sqsClient;
this.sqsQueueName = sqsQueueName;
}

@Override
public String toString() {
StringBuilder builder = new StringBuilder(super.toString());
builder.append(", flywayScriptLocationOverride=");
builder.append(flywayScriptLocationOverride);
builder.append(", sqsClient=");
builder.append(sqsClient == null ? "disabled" : "enabled");
builder.append(", sqsQueueName=");
builder.append(sqsQueueName);
return builder.toString();
}

/**
* Read configuration variables from a layered {@link ConfigLoader} and build an {@link
* AppConfiguration} instance from them.
*
* @param getenv function used to access environment variables (provided explicitly for testing)
* @return instance representing the configuration provided to this application via the
* environment variables
* @throws ConfigException if the configuration passed to the application is invalid
*/
public static AppConfiguration loadConfig() {
final var configLoader = LayeredConfiguration.createConfigLoader(Map.of(), System::getenv);
public static AppConfiguration loadConfig(Function<String, String> getenv) {
final var configLoader = LayeredConfiguration.createConfigLoader(Map.of(), getenv);

MetricOptions metricOptions = loadMetricOptions(configLoader);
DatabaseOptions databaseOptions = loadDatabaseOptions(configLoader);

String flywayScriptLocation =
configLoader.stringOptionEmptyOK(ENV_VAR_FLYWAY_SCRIPT_LOCATION).orElse("");

return new AppConfiguration(metricOptions, databaseOptions, flywayScriptLocation);
final String sqsQueueName = configLoader.stringValue(ENV_VAR_KEY_SQS_QUEUE_NAME, "");
final SqsClient sqsClient = sqsQueueName.isEmpty() ? null : createSqsClient(configLoader);

return new AppConfiguration(
metricOptions, databaseOptions, flywayScriptLocation, sqsClient, sqsQueueName);
}

/**
* Creates a {@link SqsClient} instance using settings from the provided {@link ConfigLoader}.
*
* @param configLoader used to load configuration values
* @return the {@link SqsClient}
*/
static SqsClient createSqsClient(ConfigLoader configLoader) {
final var clientBuilder = SqsClient.builder();
// either region or endpoint can be set on ssmClient but not both
if (configLoader.stringOption(ENV_VAR_KEY_SQS_ENDPOINT).isPresent()) {
// region is required when defining endpoint
clientBuilder
.region(configLoader.parsedValue(ENV_VAR_KEY_SQS_REGION, Region.class, Region::of))
.endpointOverride(URI.create(configLoader.stringValue(ENV_VAR_KEY_SQS_ENDPOINT)));
} else if (configLoader.stringOption(ENV_VAR_KEY_SQS_REGION).isPresent()) {
clientBuilder.region(
configLoader.parsedValue(ENV_VAR_KEY_SQS_REGION, Region.class, Region::of));
}
if (configLoader.stringOption(ENV_VAR_KEY_SQS_ACCESS_KEY).isPresent()) {
clientBuilder.credentialsProvider(
StaticCredentialsProvider.create(
AwsBasicCredentials.create(
configLoader.stringValue(ENV_VAR_KEY_SQS_ACCESS_KEY),
configLoader.stringValue(ENV_VAR_KEY_SQS_SECRET_KEY))));
}
return clientBuilder.build();
}
}
Loading

0 comments on commit 75ba751

Please sign in to comment.