Skip to content

Commit

Permalink
Merge pull request #524 from folio-org/MODFQMMGR-544
Browse files Browse the repository at this point in the history
MODFQMMGR-544: Remove final materialized view
  • Loading branch information
bvsharp authored Nov 5, 2024
2 parents 1a6b13b + 771240d commit 2018fe6
Show file tree
Hide file tree
Showing 10 changed files with 24 additions and 147 deletions.
31 changes: 0 additions & 31 deletions src/main/java/org/folio/fqm/repository/DataRefreshRepository.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ public class DataRefreshRepository {
public static final Field<String> CURRENCY_FIELD = field("currency", String.class);
public static final Field<Double> EXCHANGE_RATE_FIELD = field("exchange_rate", Double.class);
public static final String EXCHANGE_RATE_TABLE = "currency_exchange_rates";

private static final String REFRESH_MATERIALIZED_VIEW_CONCURRENTLY_SQL = "REFRESH MATERIALIZED VIEW CONCURRENTLY ";
private static final String REFRESH_MATERIALIZED_VIEW_SQL = "REFRESH MATERIALIZED VIEW ";
private static final String GET_EXCHANGE_RATE_PATH = "finance/exchange-rate";
private static final String GET_LOCALE_SETTINGS_PATH = "configurations/entries";
private static final Map<String, String> GET_LOCALE_SETTINGS_PARAMS = Map.of(
Expand Down Expand Up @@ -78,34 +75,6 @@ public class DataRefreshRepository {

private final SimpleHttpClient simpleHttpClient;

/**
* Refresh a list of materialized views
*
* @param tenantId Tenant ID
* @param viewsToRefresh List of materialized views to refresh
* @param refreshConcurrently Whether to execute a concurrent refresh
* @return List of all materialized views that failed to refresh
*/
public List<String> refreshMaterializedViews(String tenantId, List<String> viewsToRefresh, boolean refreshConcurrently) {
List<String> failedRefreshes = new ArrayList<>();
String refreshType = refreshConcurrently ? "concurrently" : "non-concurrently";
for (String matViewName : viewsToRefresh) {
String fullName = tenantId + "_mod_fqm_manager." + matViewName;
log.info("Refreshing materialized view {} {}", fullName, refreshType);
try {
if (refreshConcurrently) {
jooqContext.execute(REFRESH_MATERIALIZED_VIEW_CONCURRENTLY_SQL + fullName);
} else {
jooqContext.execute(REFRESH_MATERIALIZED_VIEW_SQL + fullName);
}
} catch (Exception e) {
log.info("Failed to refresh materialized view {} {}.", matViewName, refreshType);
failedRefreshes.add(matViewName);
}
}
return failedRefreshes;
}

/**
* Refresh the currency exchange rates for a tenant, based on the tenant's default system currency.
*
Expand Down
12 changes: 2 additions & 10 deletions src/main/java/org/folio/fqm/service/DataRefreshService.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,9 @@
public class DataRefreshService {
private final DataRefreshRepository dataRefreshRepository;

static final List<String> MATERIALIZED_VIEW_NAMES = List.of(
"drv_inventory_statistical_code_full"
);

public DataRefreshResponse refreshData(String tenantId) {
List<String> failedConcurrentRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, failedConcurrentRefreshes, false);
List<String> successRefreshes = new ArrayList<>(MATERIALIZED_VIEW_NAMES
.stream()
.filter(matView -> !failedRefreshes.contains(matView))
.toList());
List<String> successRefreshes = new ArrayList<>();
List<String> failedRefreshes = new ArrayList<>();
if (dataRefreshRepository.refreshExchangeRates(tenantId)) {
successRefreshes.add(EXCHANGE_RATE_TABLE);
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
<tableExists tableName = "statistical_code" schemaName="${tenant_id}_mod_inventory_storage"/>
<tableExists tableName = "statistical_code_type" schemaName="${tenant_id}_mod_inventory_storage"/>
</preConditions>
<sqlFile path="sql/create-mat-view-drv-inventory-statistical-code-full.sql" relativeToChangelogFile="true"/>
<sqlFile path="sql/create-view-drv-inventory-statistical-code-full.sql" relativeToChangelogFile="true"/>
</changeSet>

<changeSet id="MODFQMMGR-107" author="[email protected]">
Expand Down Expand Up @@ -44,10 +44,7 @@
<sql>DROP MATERIALIZED VIEW IF EXISTS drv_pol_receipt_status;</sql>
</changeSet>
<changeSet id="MODFQMMGR-210-refresh-statistical-code" author="[email protected]" runOnChange="true">
<preConditions onFail="CONTINUE">
<sqlCheck expectedResult="1">SELECT COUNT(*) FROM pg_matviews WHERE schemaname = '${tenant_id}_mod_fqm_manager'AND matviewname = 'drv_inventory_statistical_code_full';</sqlCheck>
</preConditions>
<sql>REFRESH MATERIALIZED VIEW drv_inventory_statistical_code_full;</sql>
<!-- Changeset logic intentionally removed -->
</changeSet>
<changeSet id="MODFQMMGR-210-refresh-languages" author="[email protected]" runOnChange="true">
<!-- Changeset logic intentionally removed -->
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
DROP MATERIALIZED VIEW IF EXISTS drv_inventory_statistical_code_full;
DROP MATERIALIZED VIEW IF EXISTS drv_circulation_loan_status;
DROP MATERIALIZED VIEW IF EXISTS drv_inventory_item_status;
DROP VIEW IF EXISTS drv_inventory_statistical_codes_full;

CREATE MATERIALIZED VIEW drv_inventory_statistical_code_full AS
CREATE VIEW drv_inventory_statistical_codes_full AS
SELECT
statcode.id AS "id",
CONCAT(stattype.jsonb ->> 'name', ': ', statcode.jsonb ->> 'code', ' - ', statcode.jsonb ->> 'name') AS "statistical_code"
FROM ${tenant_id}_mod_fqm_manager.src_inventory_statistical_code statcode
JOIN ${tenant_id}_mod_fqm_manager.src_inventory_statistical_code_type stattype ON stattype.id::text = statcode.jsonb ->> 'statisticalCodeTypeId'
WITH NO DATA;

CREATE UNIQUE INDEX fqm_statistical_code_full_idx
ON ${tenant_id}_mod_fqm_manager.drv_inventory_statistical_code_full(statistical_code);
JOIN ${tenant_id}_mod_fqm_manager.src_inventory_statistical_code_type stattype ON stattype.id::text = statcode.jsonb ->> 'statisticalCodeTypeId';
Original file line number Diff line number Diff line change
Expand Up @@ -333,8 +333,8 @@
idColumnName: 'statistical_code_ids',
queryable: true,
visibleByDefault: true,
valueGetter: "( SELECT array_agg(statcode.statistical_code) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_code_full statcode ON (record.value::text) = statcode.id::text)",
filterValueGetter: "( SELECT array_agg(lower(statcode.statistical_code)) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_code_full statcode ON (record.value::text) = statcode.id::text)",
valueGetter: "( SELECT array_agg(statcode.statistical_code) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_codes_full statcode ON (record.value::text) = statcode.id::text)",
filterValueGetter: "( SELECT array_agg(lower(statcode.statistical_code)) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_codes_full statcode ON (record.value::text) = statcode.id::text)",
valueFunction: 'lower(:value)',
source: {
entityTypeId: 'd2da8cc7-9171-4d3e-8aba-4da286eb5f1c',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1092,7 +1092,7 @@
idColumnName: 'statistical_code_ids',
queryable: true,
visibleByDefault: false,
valueGetter: "( SELECT array_agg(statcode.statistical_code) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_code_full statcode ON (record.value::text) = statcode.id::text)",
valueGetter: "( SELECT array_agg(statcode.statistical_code) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_codes_full statcode ON (record.value::text) = statcode.id::text)",
source: {
entityTypeId: 'd2da8cc7-9171-4d3e-8aba-4da286eb5f1c',
columnName: 'statistical_code',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
{
type: 'db',
alias: 'statistical_code_full',
target: 'drv_inventory_statistical_code_full', // materialized
target: 'drv_inventory_statistical_codes_full',
}
],
requiredPermissions: [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -596,8 +596,8 @@
queryable: true,
visibleByDefault: false,
essential: true,
valueGetter: "( SELECT array_agg(statcode.statistical_code) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_code_full statcode ON (record.value::text) = statcode.id::text)",
filterValueGetter: "( SELECT array_agg(lower(statcode.statistical_code)) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_code_full statcode ON (record.value::text) = statcode.id::text)",
valueGetter: "( SELECT array_agg(statcode.statistical_code) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_codes_full statcode ON (record.value::text) = statcode.id::text)",
filterValueGetter: "( SELECT array_agg(lower(statcode.statistical_code)) FILTER (WHERE (statcode.statistical_code) IS NOT NULL) AS array_agg FROM jsonb_array_elements_text((:sourceAlias.jsonb -> 'statisticalCodeIds'::text)) record(value) JOIN drv_inventory_statistical_codes_full statcode ON (record.value::text) = statcode.id::text)",
valueFunction: 'lower(:value)',
source: {
entityTypeId: 'd2da8cc7-9171-4d3e-8aba-4da286eb5f1c',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import org.jooq.InsertValuesStep2;
import org.jooq.Record;
import org.jooq.Record2;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.DSL;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
Expand All @@ -18,15 +17,12 @@
import org.mockito.stubbing.Answer;

import java.util.Collection;
import java.util.List;
import java.util.Map;

import static org.folio.fqm.repository.DataRefreshRepository.CURRENCY_FIELD;
import static org.folio.fqm.repository.DataRefreshRepository.EXCHANGE_RATE_FIELD;
import static org.jooq.impl.DSL.table;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.eq;
Expand All @@ -44,47 +40,6 @@ class DataRefreshRepositoryTest {
@Mock
private SimpleHttpClient simpleHttpClient;

@Test
void refreshMaterializedViewsConcurrentlyTest() {
String tenantId = "tenant_01";
List<String> viewsToRefresh = List.of("matview1", "matview2");
String expectedMatViewSql1 = "REFRESH MATERIALIZED VIEW CONCURRENTLY tenant_01_mod_fqm_manager.matview1";
String expectedMatViewSql2 = "REFRESH MATERIALIZED VIEW CONCURRENTLY tenant_01_mod_fqm_manager.matview2";
when(jooqContext.execute(anyString())).thenReturn(1);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, viewsToRefresh, true);
verify(jooqContext, times(1)).execute(expectedMatViewSql1);
verify(jooqContext, times(1)).execute(expectedMatViewSql2);
assertTrue(failedRefreshes.isEmpty());
}

@Test
void refreshMaterializedViewsTest() {
String tenantId = "tenant_01";
List<String> viewsToRefresh = List.of("matview1", "matview2");
String expectedMatViewSql1 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview1";
String expectedMatViewSql2 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview2";
when(jooqContext.execute(anyString())).thenReturn(1);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, viewsToRefresh, false);
verify(jooqContext, times(1)).execute(expectedMatViewSql1);
verify(jooqContext, times(1)).execute(expectedMatViewSql2);
assertTrue(failedRefreshes.isEmpty());
}

@Test
void shouldCatchExceptionWhenRefreshingMaterializedViews() {
String tenantId = "tenant_01";
List<String> viewsToRefresh = List.of("matview1", "matview2");
String expectedMatViewSql1 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview1";
String expectedMatViewSql2 = "REFRESH MATERIALIZED VIEW tenant_01_mod_fqm_manager.matview2";
when(jooqContext.execute(expectedMatViewSql1)).thenReturn(1);
when(jooqContext.execute(expectedMatViewSql2)).thenThrow(DataAccessException.class);
List<String> failedRefreshes = dataRefreshRepository.refreshMaterializedViews(tenantId, viewsToRefresh, false);
verify(jooqContext, times(1)).execute(expectedMatViewSql1);
verify(jooqContext, times(1)).execute(expectedMatViewSql2);
assertFalse(failedRefreshes.contains("matview1"));
assertTrue(failedRefreshes.contains("matview2"));
}

@Test
void shouldRefreshExchangeRates() {
String tenantId = "tenant_01";
Expand Down
53 changes: 9 additions & 44 deletions src/test/java/org/folio/fqm/service/DataRefreshServiceTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,7 @@
import java.util.List;

import static org.folio.fqm.repository.DataRefreshRepository.EXCHANGE_RATE_TABLE;
import static org.folio.fqm.service.DataRefreshService.MATERIALIZED_VIEW_NAMES;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
Expand All @@ -28,51 +25,19 @@ class DataRefreshServiceTest {
@Test
void refreshDataTest() {
String tenantId = "tenant_01";
List<String> expectedSuccessRefreshViews = new ArrayList<>(MATERIALIZED_VIEW_NAMES);
expectedSuccessRefreshViews.add(EXCHANGE_RATE_TABLE);
DataRefreshResponse expectedDataRefreshResponse = new DataRefreshResponse()
.successfulRefresh(expectedSuccessRefreshViews)
List<String> refreshList = new ArrayList<>(List.of(EXCHANGE_RATE_TABLE));
DataRefreshResponse successRefresh = new DataRefreshResponse()
.successfulRefresh(refreshList)
.failedRefresh(List.of());
when(dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true)).thenReturn(List.of());
DataRefreshResponse failedRefresh = new DataRefreshResponse()
.successfulRefresh(List.of())
.failedRefresh(refreshList);
when(dataRefreshRepository.refreshExchangeRates(tenantId)).thenReturn(true);
DataRefreshResponse actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
assertEquals(expectedDataRefreshResponse, actualDataRefreshResponse);
}
assertEquals(successRefresh, actualDataRefreshResponse);

@Test
void shouldRetryRefreshIfConcurrentRefreshFails() {
String tenantId = "tenant_01";
List<String> expectedSuccessRefreshViews = new ArrayList<>(MATERIALIZED_VIEW_NAMES);
expectedSuccessRefreshViews.add(EXCHANGE_RATE_TABLE);
DataRefreshResponse expectedDataRefreshResponse = new DataRefreshResponse()
.successfulRefresh(expectedSuccessRefreshViews)
.failedRefresh(List.of());
when(dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true)).thenReturn(List.of("drv_languages"));
when(dataRefreshRepository.refreshMaterializedViews(tenantId, List.of("drv_languages"), false)).thenReturn(List.of());
when(dataRefreshRepository.refreshExchangeRates(tenantId)).thenReturn(true);
DataRefreshResponse actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
verify(dataRefreshRepository, times(1)).refreshMaterializedViews(tenantId, List.of("drv_languages"), false);
assertEquals(expectedDataRefreshResponse, actualDataRefreshResponse);
}

@Test
void shouldReturnFailedRefreshes() {
String tenantId = "tenant_01";
List<String> expectedSuccessRefreshes = MATERIALIZED_VIEW_NAMES
.stream()
.filter(view -> !view.equals("drv_languages"))
.toList();
List<String> expectedFailedRefreshes = List.of("drv_languages", "currency_exchange_rates");
DataRefreshResponse expectedDataRefreshResponse = new DataRefreshResponse()
.successfulRefresh(expectedSuccessRefreshes)
.failedRefresh(expectedFailedRefreshes);
when(dataRefreshRepository.refreshMaterializedViews(tenantId, MATERIALIZED_VIEW_NAMES, true))
.thenReturn(new ArrayList<>(List.of("drv_languages")));
when(dataRefreshRepository.refreshMaterializedViews(tenantId, List.of("drv_languages"), false))
.thenReturn(new ArrayList<>(List.of("drv_languages")));
when(dataRefreshRepository.refreshExchangeRates(tenantId)).thenReturn(false);
DataRefreshResponse actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
verify(dataRefreshRepository, times(1)).refreshMaterializedViews(tenantId, List.of("drv_languages"), false);
assertEquals(expectedDataRefreshResponse, actualDataRefreshResponse);
actualDataRefreshResponse = dataRefreshService.refreshData(tenantId);
assertEquals(failedRefresh, actualDataRefreshResponse);
}
}

0 comments on commit 2018fe6

Please sign in to comment.