Skip to content

Commit

Permalink
MODFQMMGR-331: Retry materialized view refreshes on failure (#295)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvsharp
bvsharp authored Jun 27, 2024
1 parent 82ab3ca commit 661f08b
Show file tree
Hide file tree
Showing 7 changed files with 188 additions and 40 deletions.
56 changes: 38 additions & 18 deletions src/main/java/org/folio/fqm/repository/DataRefreshRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,17 @@ public class DataRefreshRepository {

public static final Field<String> CURRENCY_FIELD = field("currency", String.class);
public static final Field<Double> EXCHANGE_RATE_FIELD = field("exchange_rate", Double.class);
public static final String EXCHANGE_RATE_TABLE = "currency_exchange_rates";

private static final String REFRESH_MATERIALIZED_VIEW_SQL = "REFRESH MATERIALIZED VIEW CONCURRENTLY ";
private static final String REFRESH_MATERIALIZED_VIEW_CONCURRENTLY_SQL = "REFRESH MATERIALIZED VIEW CONCURRENTLY ";
private static final String REFRESH_MATERIALIZED_VIEW_SQL = "REFRESH MATERIALIZED VIEW ";
private static final String GET_EXCHANGE_RATE_PATH = "finance/exchange-rate";
private static final String GET_LOCALE_SETTINGS_PATH = "configurations/entries";
private static final Map<String, String> GET_LOCALE_SETTINGS_PARAMS = Map.of(
"query", "(module==ORG and configName==localeSettings)"
);

private static final List<String> MATERIALIZED_VIEW_NAMES = List.of(
"drv_circulation_loan_status",
"drv_inventory_item_status",
"drv_pol_payment_status",
"drv_pol_receipt_status",
"drv_inventory_statistical_code_full",
"drv_languages"
);

static final List<String> SYSTEM_SUPPORTED_CURRENCIES = List.of(
private static final List<String> SYSTEM_SUPPORTED_CURRENCIES = List.of(
"USD",
"EUR",
"JPY",
Expand Down Expand Up @@ -85,21 +78,47 @@ public class DataRefreshRepository {

private final SimpleHttpClient simpleHttpClient;

public void refreshMaterializedViews(String tenantId) {
for (String matViewName : MATERIALIZED_VIEW_NAMES) {
/**
* Refresh a list of materialized views
*
* @param tenantId Tenant ID
* @param viewsToRefresh List of materialized views to refresh
* @param refreshConcurrently Whether to execute a concurrent refresh
* @return List of all materialized views that failed to refresh
*/
public List<String> refreshMaterializedViews(String tenantId, List<String> viewsToRefresh, boolean refreshConcurrently) {
List<String> failedRefreshes = new ArrayList<>();
String refreshType = refreshConcurrently ? "concurrently" : "non-concurrently";
for (String matViewName : viewsToRefresh) {
String fullName = tenantId + "_mod_fqm_manager." + matViewName;
log.info("Refreshing materialized view {}", fullName);
jooqContext.execute(REFRESH_MATERIALIZED_VIEW_SQL + fullName);
log.info("Refreshing materialized view {} {}", fullName, refreshType);
try {
if (refreshConcurrently) {
jooqContext.execute(REFRESH_MATERIALIZED_VIEW_CONCURRENTLY_SQL + fullName);
} else {
jooqContext.execute(REFRESH_MATERIALIZED_VIEW_SQL + fullName);
}
} catch (Exception e) {
log.info("Failed to refresh materialized view {} {}.", matViewName, refreshType);
failedRefreshes.add(matViewName);
}
}
return failedRefreshes;
}

public void refreshExchangeRates(String tenantId) {
/**
* Refresh the currency exchange rates for a tenant, based on the tenant's default system currency.
*
* @param tenantId Tenant ID
* @return True if refresh successful, false otherwise
*/
public boolean refreshExchangeRates(String tenantId) {
log.info("Refreshing exchange rates");
String fullTableName = tenantId + "_mod_fqm_manager.currency_exchange_rates";
String fullTableName = tenantId + "_mod_fqm_manager." + EXCHANGE_RATE_TABLE;
String systemCurrency = getSystemCurrencyCode();
if (!SYSTEM_SUPPORTED_CURRENCIES.contains(systemCurrency)) {
log.info("System currency does not support automatic exchange rate calculation");
return;
return false;
}

List<Record2<String, Double>> exchangeRates = new ArrayList<>();
Expand All @@ -120,6 +139,7 @@ public void refreshExchangeRates(String tenantId) {
.doUpdate()
.set(EXCHANGE_RATE_FIELD, DSL.field("EXCLUDED." + EXCHANGE_RATE_FIELD.getName(), Double.class))
.execute();
return true;
}

private String getSystemCurrencyCode() {
Expand Down
7 changes: 3 additions & 4 deletions src/main/java/org/folio/fqm/resource/DataRefreshController.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
package org.folio.fqm.resource;

import lombok.RequiredArgsConstructor;
import org.folio.fqm.domain.dto.DataRefreshResponse;
import org.folio.fqm.service.DataRefreshService;
import org.folio.spring.FolioExecutionContext;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RestController;

Expand All @@ -14,8 +14,7 @@ public class DataRefreshController implements MaterializedViewsApi {
private final DataRefreshService dataRefreshService;

@Override
public ResponseEntity<Void> refreshData() {
dataRefreshService.refreshData(executionContext.getTenantId());
return new ResponseEntity<>(HttpStatus.NO_CONTENT);
public ResponseEntity<DataRefreshResponse> refreshData() {
return ResponseEntity.ok(dataRefreshService.refreshData(executionContext.getTenantId()));
}
}
33 changes: 30 additions & 3 deletions src/main/java/org/folio/fqm/service/DataRefreshService.java
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,43 @@
package org.folio.fqm.service;

import lombok.RequiredArgsConstructor;
import org.folio.fqm.domain.dto.DataRefreshResponse;
import org.folio.fqm.repository.DataRefreshRepository;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.List;

import static org.folio.fqm.repository.DataRefreshRepository.EXCHANGE_RATE_TABLE;

@Service
@RequiredArgsConstructor
public class DataRefreshService {
private final DataRefreshRepository dataRefreshRepository;

public void refreshData(String tenantId) {
dataRefreshRepository.refreshMaterializedViews(tenantId);
dataRefreshRepository.refreshExchangeRates(tenantId);
static final List<String> MATERIALIZED_VIEW_NAMES = List.of(
"drv_circulation_loan_status",
"drv_inventory_item_status",
"drv_pol_payment_status",
"drv_pol_receipt_status",
"drv_inventory_statistical_code_full",
"drv_languages"
);

public DataRefreshResponse refreshData(String tenantId) {
List<String> failedConcurrentRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, failedConcurrentRefreshes, false);
List<String> successRefreshes = new ArrayList<>(MATERIALIZED_VIEW_NAMES
.stream()
.filter(matView -> !failedRefreshes.contains(matView))
.toList());
if (dataRefreshRepository.refreshExchangeRates(tenantId)) {
successRefreshes.add(EXCHANGE_RATE_TABLE);
} else {
failedRefreshes.add(EXCHANGE_RATE_TABLE);
}
return new DataRefreshResponse()
.successfulRefresh(successRefreshes)
.failedRefresh(failedRefreshes);
}
}
17 changes: 16 additions & 1 deletion src/main/resources/swagger.api/mod-fqm-manager.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,12 @@ paths:
- materializedViews
description: Refresh all materialized views and similar data for a tenant.
responses:
'204':
'200':
description: 'Data refreshed'
content:
application/json:
schema:
$ref: '#/components/schemas/dataRefreshResponse'
'400':
$ref: '#/components/responses/badRequestResponse'
'500':
Expand Down Expand Up @@ -106,6 +110,17 @@ components:
$ref: schemas/EntityTypeSummaryDTO.json
purgedQueries:
$ref: schemas/PurgedQueries.json
dataRefreshResponse:
type: object
properties:
successfulRefresh:
type: array
items:
type: string
failedRefresh:
type: array
items:
type: string

responses:
badRequestResponse:
Expand Down
11 changes: 8 additions & 3 deletions src/test/java/org/folio/fqm/controller/DataRefreshControllerTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.folio.fqm.controller;

import org.folio.fqm.domain.dto.DataRefreshResponse;
import org.folio.fqm.resource.DataRefreshController;
import org.folio.fqm.service.DataRefreshService;
import org.folio.spring.FolioExecutionContext;
Expand All @@ -12,7 +13,8 @@
import org.springframework.test.web.servlet.RequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;

import static org.mockito.Mockito.doNothing;
import java.util.List;

import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
Expand All @@ -31,13 +33,16 @@ class DataRefreshControllerTest {
@Test
void refreshDataTest() throws Exception {
String tenantId = "tenant_01";
DataRefreshResponse expectedResponse = new DataRefreshResponse()
.successfulRefresh(List.of())
.failedRefresh(List.of());
RequestBuilder requestBuilder = MockMvcRequestBuilders.post("/entity-types/materialized-views/refresh")
.header(XOkapiHeaders.TENANT, tenantId)
.contentType(APPLICATION_JSON);
when(executionContext.getTenantId()).thenReturn(tenantId);
doNothing().when(dataRefreshService).refreshData(tenantId);
when(dataRefreshService.refreshData(tenantId)).thenReturn(expectedResponse);
mockMvc.perform(requestBuilder)
.andExpect(status().isNoContent());
.andExpect(status().isOk());
verify(dataRefreshService, times(1)).refreshData(tenantId);
}
}
44 changes: 39 additions & 5 deletions src/test/java/org/folio/fqm/repository/DataRefreshRepositoryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import org.jooq.InsertValuesStep2;
import org.jooq.Record;
import org.jooq.Record2;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.DSL;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
Expand All @@ -17,12 +18,15 @@
import org.mockito.stubbing.Answer;

import java.util.Collection;
import java.util.List;
import java.util.Map;

import static org.folio.fqm.repository.DataRefreshRepository.CURRENCY_FIELD;
import static org.folio.fqm.repository.DataRefreshRepository.EXCHANGE_RATE_FIELD;
import static org.jooq.impl.DSL.table;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.eq;
Expand All @@ -40,15 +44,45 @@ class DataRefreshRepositoryTest {
@Mock
private SimpleHttpClient simpleHttpClient;

@Test
void refreshMaterializedViewsConcurrentlyTest() {
String tenantId = "tenant_01";
List<String> viewsToRefresh = List.of("matview1", "matview2");
String expectedMatViewSql1 = "REFRESH MATERIALIZED VIEW CONCURRENTLY tenant_01_mod_fqm_manager.matview1";
String expectedMatViewSql2 = "REFRESH MATERIALIZED VIEW CONCURRENTLY tenant_01_mod_fqm_manager.matview2";
when(jooqContext.execute(anyString())).thenReturn(1);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, viewsToRefresh, true);
verify(jooqContext, times(1)).execute(expectedMatViewSql1);
verify(jooqContext, times(1)).execute(expectedMatViewSql2);
assertTrue(failedRefreshes.isEmpty());
}

@Test
void refreshMaterializedViewsTest() {
String tenantId = "tenant_01";
String expectedItemStatusSql = "REFRESH MATERIALIZED VIEW CONCURRENTLY tenant_01_mod_fqm_manager.drv_inventory_item_status";
String expectedLoanStatusSql = "REFRESH MATERIALIZED VIEW CONCURRENTLY tenant_01_mod_fqm_manager.drv_circulation_loan_status";
List<String> viewsToRefresh = List.of("matview1", "matview2");
String expectedMatViewSql1 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview1";
String expectedMatViewSql2 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview2";
when(jooqContext.execute(anyString())).thenReturn(1);
dataRefreshRepository.refreshMaterializedViews(tenantId);
verify(jooqContext, times(1)).execute(expectedItemStatusSql);
verify(jooqContext, times(1)).execute(expectedLoanStatusSql);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, viewsToRefresh, false);
verify(jooqContext, times(1)).execute(expectedMatViewSql1);
verify(jooqContext, times(1)).execute(expectedMatViewSql2);
assertTrue(failedRefreshes.isEmpty());
}

@Test
void shouldCatchExceptionWhenRefreshingMaterializedViews() {
String tenantId = "tenant_01";
List<String> viewsToRefresh = List.of("matview1", "matview2");
String expectedMatViewSql1 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview1";
String expectedMatViewSql2 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview2";
when(jooqContext.execute(expectedMatViewSql1)).thenReturn(1);
when(jooqContext.execute(expectedMatViewSql2)).thenThrow(DataAccessException.class);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, viewsToRefresh, false);
verify(jooqContext, times(1)).execute(expectedMatViewSql1);
verify(jooqContext, times(1)).execute(expectedMatViewSql2);
assertFalse(failedRefreshes.contains("matview1"));
assertTrue(failedRefreshes.contains("matview2"));
}

@Test
Expand Down
60 changes: 54 additions & 6 deletions src/test/java/org/folio/fqm/service/DataRefreshServiceTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,22 @@
package org.folio.fqm.service;

import org.folio.fqm.domain.dto.DataRefreshResponse;
import org.folio.fqm.repository.DataRefreshRepository;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

import static org.mockito.Mockito.doNothing;
import java.util.ArrayList;
import java.util.List;

import static org.folio.fqm.repository.DataRefreshRepository.EXCHANGE_RATE_TABLE;
import static org.folio.fqm.service.DataRefreshService.MATERIALIZED_VIEW_NAMES;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
class DataRefreshServiceTest {
Expand All @@ -21,10 +28,51 @@ class DataRefreshServiceTest {
@Test
void refreshDataTest() {
String tenantId = "tenant_01";
doNothing().when(dataRefreshRepository).refreshMaterializedViews(tenantId);
doNothing().when(dataRefreshRepository).refreshExchangeRates(tenantId);
dataRefreshService.refreshData(tenantId);
verify(dataRefreshRepository, times(1)).refreshMaterializedViews(tenantId);
verify(dataRefreshRepository, times(1)).refreshExchangeRates(tenantId);
List<String> expectedSuccessRefreshViews = new ArrayList<>(MATERIALIZED_VIEW_NAMES);
expectedSuccessRefreshViews.add(EXCHANGE_RATE_TABLE);
DataRefreshResponse expectedDataRefreshResponse = new DataRefreshResponse()
.successfulRefresh(expectedSuccessRefreshViews)
.failedRefresh(List.of());
when(dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true)).thenReturn(List.of());
when(dataRefreshRepository.refreshExchangeRates(tenantId)).thenReturn(true);
DataRefreshResponse actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
assertEquals(expectedDataRefreshResponse, actualDataRefreshResponse);
}

@Test
void shouldRetryRefreshIfConcurrentRefreshFails() {
String tenantId = "tenant_01";
List<String> expectedSuccessRefreshViews = new ArrayList<>(MATERIALIZED_VIEW_NAMES);
expectedSuccessRefreshViews.add(EXCHANGE_RATE_TABLE);
DataRefreshResponse expectedDataRefreshResponse = new DataRefreshResponse()
.successfulRefresh(expectedSuccessRefreshViews)
.failedRefresh(List.of());
when(dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true)).thenReturn(List.of("drv_languages"));
when(dataRefreshRepository.refreshMaterializedViews(tenantId, List.of("drv_languages"), false)).thenReturn(List.of());
when(dataRefreshRepository.refreshExchangeRates(tenantId)).thenReturn(true);
DataRefreshResponse actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
verify(dataRefreshRepository, times(1)).refreshMaterializedViews(tenantId, List.of("drv_languages"), false);
assertEquals(expectedDataRefreshResponse, actualDataRefreshResponse);
}

@Test
void shouldReturnFailedRefreshes() {
String tenantId = "tenant_01";
List<String> expectedSuccessRefreshes = MATERIALIZED_VIEW_NAMES
.stream()
.filter(view -> !view.equals("drv_languages"))
.toList();
List<String> expectedFailedRefreshes = List.of("drv_languages", "currency_exchange_rates");
DataRefreshResponse expectedDataRefreshResponse = new DataRefreshResponse()
.successfulRefresh(expectedSuccessRefreshes)
.failedRefresh(expectedFailedRefreshes);
when(dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true))
.thenReturn(new ArrayList<>(List.of("drv_languages")));
when(dataRefreshRepository.refreshMaterializedViews(tenantId, List.of("drv_languages"), false))
.thenReturn(new ArrayList<>(List.of("drv_languages")));
when(dataRefreshRepository.refreshExchangeRates(tenantId)).thenReturn(false);
DataRefreshResponse actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
verify(dataRefreshRepository, times(1)).refreshMaterializedViews(tenantId, List.of("drv_languages"), false);
assertEquals(expectedDataRefreshResponse, actualDataRefreshResponse);
}
}

0 comments on commit 661f08b

Please sign in to comment.