Skip to content

Commit

Permalink
MODFQMMGR-468: Aggregate Tenant locations across all tenants
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvsharp
bvsharp committed Oct 4, 2024
1 parent 9ab3bcd commit 4997dc2
Show file tree
Hide file tree
Showing 10 changed files with 103 additions and 39 deletions.
12 changes: 12 additions & 0 deletions src/main/java/org/folio/fqm/client/SimpleHttpClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import org.springframework.stereotype.Service;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestHeader;

import java.util.Map;

Expand All @@ -21,4 +22,15 @@ public interface SimpleHttpClient {
*/
@GetMapping(value = "/{path}?{queryParams}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams);

/**
* Retrieve arbitrary data from a FOLIO API endpoint for the specified tenant.
*
* @param path - the path of the API endpoint
* @param queryParams - a map of query parameters to pass to the API endpoint
* @param tenant - FOLIO tenant from which to retrieve data
* @return the body of the response (JSON)
*/
@GetMapping(value = "/{path}?{queryParams}", produces = MediaType.APPLICATION_JSON_VALUE)
String get(@PathVariable String path, @SpringQueryMap Map<String, String> queryParams, @RequestHeader("X-Okapi-Tenant") String tenant);
}
2 changes: 1 addition & 1 deletion src/main/java/org/folio/fqm/repository/IdStreamer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public int streamIdsInBatch(EntityType entityType,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
boolean ecsEnabled = crossTenantQueryService.ecsEnabled();
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return this.streamIdsInBatch(entityType, sortResults, fql, batchSize, idsConsumer, tenantsToQuery, ecsEnabled);
}

Expand Down
32 changes: 26 additions & 6 deletions src/main/java/org/folio/fqm/service/CrossTenantQueryService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,35 @@ public class CrossTenantQueryService {

private static final String COMPOSITE_INSTANCES_ID = "6b08439b-4f8e-4468-8046-ea620f5cfb74";
private static final String SIMPLE_INSTANCES_ID = "8fc4a9d2-7ccf-4233-afb8-796911839862";
private static final String SIMPLE_INSTANCE_STATUS_ID = "9c239bfd-198f-4013-bbc4-4551c0cbdeaa";
private static final String SIMPLE_INSTANCE_TYPE_ID = "af44e2e0-12e0-4eec-b80d-49feb33a866c";
private static final List<String> INSTANCE_RELATED_ENTITIES = List.of(SIMPLE_INSTANCES_ID, COMPOSITE_INSTANCES_ID, SIMPLE_INSTANCE_STATUS_ID, SIMPLE_INSTANCE_TYPE_ID);

public List<String> getTenantsToQuery(EntityType entityType, boolean forceCrossTenantQuery) {
if (!forceCrossTenantQuery
&& !Boolean.TRUE.equals(entityType.getCrossTenantQueriesEnabled())
/**
* Retrieve list of tenants to run query against.
* @param entityType Entity type definition
* @return List of tenants to query
*/
public List<String> getTenantsToQuery(EntityType entityType) {
if (!Boolean.TRUE.equals(entityType.getCrossTenantQueriesEnabled())
&& !COMPOSITE_INSTANCES_ID.equals(entityType.getId())) {
return List.of(executionContext.getTenantId());
}
// Get the ECS tenant info first, since this comes from mod-users and should work in non-ECS environments
// We can use this for determining if it's an ECS environment, and if so, retrieving the consortium ID and central tenant ID
return getTenants(entityType);
}

/**
* Retrieve list of tenants to retrieve column values from. This method skips the cross-tenant query check, since the
* column values API uses simple entity type definitions, which don't have cross-tenant queries enabled.
* method skips the cross-tenant query check
* @param entityType Entity type definition
* @return List of tenants to query
*/
public List<String> getTenantsToQueryForColumnValues(EntityType entityType) {
return getTenants(entityType);
}

private List<String> getTenants(EntityType entityType) {
Map<String, String> ecsTenantInfo = getEcsTenantInfo();
if (!ecsEnabled(ecsTenantInfo)) {
return List.of(executionContext.getTenantId());
Expand All @@ -46,7 +66,7 @@ public List<String> getTenantsToQuery(EntityType entityType, boolean forceCrossT
// The Instances entity type is required to retrieve shared instances from the central tenant when
// running queries from member tenants. This means that if we are running a query for Instances, we need to
// query the current tenant (for local records) as well as the central tenant (for shared records).
if (COMPOSITE_INSTANCES_ID.equals(entityType.getId()) || SIMPLE_INSTANCES_ID.equals(entityType.getId())) {
if (INSTANCE_RELATED_ENTITIES.contains(entityType.getId())) {
return List.of(executionContext.getTenantId(), centralTenantId);
}
return List.of(executionContext.getTenantId());
Expand Down
41 changes: 27 additions & 14 deletions src/main/java/org/folio/fqm/service/EntityTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import feign.FeignException;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.fql.model.field.FqlField;
import org.folio.fql.service.FqlValidationService;
import org.folio.fqm.client.SimpleHttpClient;
Expand All @@ -28,6 +30,7 @@

@Service
@RequiredArgsConstructor
@Log4j2
public class EntityTypeService {

private static final int COLUMN_VALUE_DEFAULT_PAGE_SIZE = 1000;
Expand Down Expand Up @@ -125,7 +128,8 @@ public ColumnValues getFieldValues(UUID entityTypeId, String fieldName, @Nullabl
}

if (field.getValueSourceApi() != null) {
return getFieldValuesFromApi(field, searchText);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQueryForColumnValues(entityType);
return getFieldValuesFromApi(field, searchText, tenantsToQuery);
}

if (field.getSource() != null) {
Expand Down Expand Up @@ -154,7 +158,7 @@ public ColumnValues getFieldValues(UUID entityTypeId, String fieldName, @Nullabl
}

private ColumnValues getTenantIds(EntityType entityType) {
List<String> tenants = crossTenantQueryService.getTenantsToQuery(entityType, true);
List<String> tenants = crossTenantQueryService.getTenantsToQueryForColumnValues(entityType);
List<ValueWithLabel> tenantValues = tenants
.stream()
.map(tenant -> new ValueWithLabel().value(tenant).label(tenant))
Expand All @@ -173,20 +177,29 @@ private ColumnValues getFieldValuesFromEntityTypeDefinition(Field field, String
return new ColumnValues().content(filteredValues);
}

private ColumnValues getFieldValuesFromApi(Field field, String searchText) {
String rawJson = fieldValueClient.get(field.getValueSourceApi().getPath(), Map.of("limit", String.valueOf(COLUMN_VALUE_DEFAULT_PAGE_SIZE)));
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read(field.getValueSourceApi().getValueJsonPath());
List<String> labels = parsedJson.read(field.getValueSourceApi().getLabelJsonPath());

List<ValueWithLabel> results = new ArrayList<>(values.size());
for (int i = 0; i < values.size(); i++) {
String value = values.get(i);
String label = labels.get(i);
if (label.contains(searchText)) {
results.add(new ValueWithLabel().value(value).label(label));
private ColumnValues getFieldValuesFromApi(Field field, String searchText, List<String> tenantsToQuery) {
Set<ValueWithLabel> resultSet = new HashSet<>();
for (String tenantId : tenantsToQuery) {
try {
String rawJson = fieldValueClient.get(field.getValueSourceApi().getPath(), Map.of("limit", String.valueOf(COLUMN_VALUE_DEFAULT_PAGE_SIZE)), tenantId);
DocumentContext parsedJson = JsonPath.parse(rawJson);
List<String> values = parsedJson.read(field.getValueSourceApi().getValueJsonPath());
List<String> labels = parsedJson.read(field.getValueSourceApi().getLabelJsonPath());
for (int i = 0; i < values.size(); i++) {
String value = values.get(i);
String label = labels.get(i);
if (label.contains(searchText)) {
resultSet.add(new ValueWithLabel().value(value).label(label));
}
}
} catch (FeignException.Unauthorized e) {
log.error("Failed to get column values from {} tenant due to exception: {}", tenantId, e.getMessage());
}
}


List<ValueWithLabel> results = new ArrayList<>(resultSet);
log.info("Results: {}", results);
results.sort(Comparator.comparing(ValueWithLabel::getLabel, String.CASE_INSENSITIVE_ORDER));
return new ColumnValues().content(results);
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/folio/fqm/service/QueryManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,15 +191,15 @@ public List<Map<String, Object>> getContents(UUID entityTypeId, List<String> fie
fields.add(colName);
}
});
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return resultSetService.getResultSet(entityTypeId, fields, ids, tenantsToQuery);
}

private List<Map<String, Object>> getContents(UUID queryId, UUID entityTypeId, List<String> fields, boolean includeResults, int offset, int limit) {
if (includeResults) {
EntityType entityType = entityTypeService.getEntityTypeDefinition(entityTypeId, true, false);
List<List<String>> resultIds = queryResultsRepository.getQueryResultIds(queryId, offset, limit);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return resultSetService.getResultSet(entityTypeId, fields, resultIds, tenantsToQuery);
}
return List.of();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/folio/fqm/service/QueryProcessorService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void getIdsInBatch(FqlQueryWithContext fqlQueryWithContext,
public List<Map<String, Object>> processQuery(EntityType entityType, String fqlQuery, List<String> fields, List<String> afterId, Integer limit) {
Fql fql = fqlService.getFql(fqlQuery);
boolean ecsEnabled = crossTenantQueryService.ecsEnabled();
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(entityType);
return resultSetRepository.getResultSetSync(
UUID.fromString(entityType.getId()),
fql,
Expand Down
30 changes: 22 additions & 8 deletions src/test/java/org/folio/fqm/service/CrossTenantQueryServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
class CrossTenantQueryServiceTest {

private static final EntityType entityType = new EntityType()
.id(UUID.randomUUID().toString())
.crossTenantQueriesEnabled(true);

@Mock
Expand Down Expand Up @@ -102,7 +103,7 @@ void shouldGetListOfTenantsToQuery() {
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -112,7 +113,7 @@ void shouldRunIntraTenantQueryForNonInstanceEntityTypes() {

List<String> expectedTenants = List.of("tenant_01");
when(executionContext.getTenantId()).thenReturn("tenant_01");
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(nonEcsEntityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(nonEcsEntityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -122,7 +123,7 @@ void shouldRunIntraTenantQueryForNonCentralTenant() {
List<String> expectedTenants = List.of(tenantId);
when(executionContext.getTenantId()).thenReturn(tenantId); // Central is tenant_01
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -132,7 +133,7 @@ void shouldRunIntraTenantQueryIfExceptionIsThrown() {
List<String> expectedTenants = List.of(tenantId);
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -144,7 +145,7 @@ void shouldReturnTenantIdOnlyIfUserTenantsApiThrowsException() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -156,7 +157,7 @@ void shouldAttemptCrossTenantQueryIfForceParamIsTrue() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO_FOR_NON_ECS_ENV);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, true);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
verify(userTenantService, times(1)).getUserTenantsResponse(tenantId);
assertEquals(expectedTenants, actualTenants);
}
Expand All @@ -171,7 +172,7 @@ void shouldNotQueryTenantIfUserLacksTenantPermissions() {
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);
doNothing().when(permissionsService).verifyUserHasNecessaryPermissions("tenant_02", entityType, true);
doThrow(MissingPermissionsException.class).when(permissionsService).verifyUserHasNecessaryPermissions("tenant_03", entityType, true);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand All @@ -186,7 +187,20 @@ void shouldQueryCentralTenantForSharedCompositeInstances() {
when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);

List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(instanceEntityType, false);
List<String> actualTenants = crossTenantQueryService.getTenantsToQuery(instanceEntityType);
assertEquals(expectedTenants, actualTenants);
}

@Test
void shouldGetListOfTenantsToQueryForColumnValues() {
String tenantId = "tenant_01";
List<String> expectedTenants = List.of("tenant_01", "tenant_02", "tenant_03");

when(executionContext.getTenantId()).thenReturn(tenantId);
when(userTenantService.getUserTenantsResponse(tenantId)).thenReturn(ECS_TENANT_INFO);
when(ecsClient.get(eq("consortia/bdaa4720-5e11-4632-bc10-d4455cf252df/user-tenants"), anyMap())).thenReturn(USER_TENANT_JSON);

List<String> actualTenants = crossTenantQueryService.getTenantsToQueryForColumnValues(entityType);
assertEquals(expectedTenants, actualTenants);
}

Expand Down
11 changes: 8 additions & 3 deletions src/test/java/org/folio/fqm/service/EntityTypeServiceTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.folio.fqm.service;

import feign.FeignException;
import org.folio.fqm.client.SimpleHttpClient;
import org.folio.fqm.repository.EntityTypeRepository;
import org.folio.fqm.testutil.TestDataFixture;
Expand Down Expand Up @@ -329,6 +330,7 @@ void shouldRemoveDuplicatePredefinedValues() {
void shouldReturnValuesFromApi() {
UUID entityTypeId = UUID.randomUUID();
String valueColumnName = "column_name";
List<String> tenantList = List.of("tenant_01", "tenant_02");
EntityType entityType = new EntityType()
.id(entityTypeId.toString())
.name("the entity type")
Expand All @@ -341,8 +343,9 @@ void shouldReturnValuesFromApi() {
)
));

when(crossTenantQueryService.getTenantsToQueryForColumnValues(entityType)).thenReturn(tenantList);
when(entityTypeFlatteningService.getFlattenedEntityType(entityTypeId, null)).thenReturn(entityType);
when(simpleHttpClient.get(eq("fake-path"), anyMap())).thenReturn("""
when(simpleHttpClient.get(eq("fake-path"), anyMap(), eq("tenant_01"))).thenReturn("""
{
"what": {
"ever": {
Expand All @@ -364,6 +367,7 @@ void shouldReturnValuesFromApi() {
}
}
""");
when(simpleHttpClient.get(eq("fake-path"), anyMap(), eq("tenant_02"))).thenThrow(FeignException.Unauthorized.class);

ColumnValues actualColumnValueLabel = entityTypeService.getFieldValues(entityTypeId, valueColumnName, "r");

Expand Down Expand Up @@ -459,6 +463,7 @@ void shouldReturnCurrencies() {
void shouldReturnTenantId() {
UUID entityTypeId = UUID.randomUUID();
String valueColumnName = "this_is_a_tenant_id_column";
List<String> tenantList = List.of("tenant1", "tenant2");
EntityType entityType = new EntityType()
.id(entityTypeId.toString())
.name("tenant-id-test")
Expand All @@ -470,7 +475,7 @@ void shouldReturnTenantId() {
));

when(entityTypeFlatteningService.getFlattenedEntityType(entityTypeId, null)).thenReturn(entityType);
when(crossTenantQueryService.getTenantsToQuery(entityType, true)).thenReturn(List.of("tenant1", "tenant2"));
when(crossTenantQueryService.getTenantsToQueryForColumnValues(entityType)).thenReturn(tenantList);

List<ValueWithLabel> actualColumnValues = entityTypeService
.getFieldValues(entityTypeId, valueColumnName, "")
Expand All @@ -496,7 +501,7 @@ void shouldIncludeCentralTenantIdInResponseForSimpleInstanceEntityType() {
));

when(entityTypeFlatteningService.getFlattenedEntityType(entityTypeId, null)).thenReturn(entityType);
when(crossTenantQueryService.getTenantsToQuery(entityType, true)).thenReturn(List.of("tenant1", "central"));
when(crossTenantQueryService.getTenantsToQueryForColumnValues(entityType)).thenReturn(List.of("tenant1", "central"));

List<ValueWithLabel> actualColumnValues = entityTypeService
.getFieldValues(entityTypeId, valueColumnName, "")
Expand Down
Loading

0 comments on commit 4997dc2

Please sign in to comment.