diff --git a/NEWS.md b/NEWS.md index debaf9836..06f848dfc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -6,7 +6,7 @@ * Endpoint `GET /search/consortium/batch/holdings` requires `consortium-search.holdings.batch.collection.get` permission ### New APIs versions -* Provides `indices v0.8` +* Provides `indices v1.0` * Provides `search v1.3` * Provides `consortium-search v2.1` * Provides `browse v1.4` @@ -49,6 +49,7 @@ * Implement new re-index flow for instance records ([MSEARCH-793](https://folio-org.atlassian.net/issues/MSEARCH-793), [MSEARCH-794](https://folio-org.atlassian.net/issues/MSEARCH-794), [MSEARCH-796](https://folio-org.atlassian.net/issues/MSEARCH-796), [MSEARCH-797](https://folio-org.atlassian.net/issues/MSEARCH-797), [MSEARCH-798](https://folio-org.atlassian.net/issues/MSEARCH-798), [MSEARCH-799](https://folio-org.atlassian.net/issues/MSEARCH-799), [MSEARCH-800](https://folio-org.atlassian.net/issues/MSEARCH-800), [MSEARCH-801](https://folio-org.atlassian.net/issues/MSEARCH-801), [MSEARCH-802](https://folio-org.atlassian.net/issues/MSEARCH-802)) * Implement Linked Data HUB index and search API ([MSEARCH-844](https://folio-org.atlassian.net/browse/MSEARCH-844)) * Extend consortium library, campus, institution API with id param ([MSEARCH-855](https://folio-org.atlassian.net/browse/MSEARCH-855)) +* Extend instance-records reindex endpoint with index settings ([MSEARCH-853](https://folio-org.atlassian.net/browse/MSEARCH-853)) ### Bug fixes * Do not delete kafka topics if collection topic is enabled ([MSEARCH-725](https://folio-org.atlassian.net/browse/MSEARCH-725)) diff --git a/README.md b/README.md index 135944380..4c37bf7c0 100644 --- a/README.md +++ b/README.md @@ -323,12 +323,13 @@ x-okapi-token: [JWT_TOKEN] { "recreateIndex": true, - "resourceName": "instance" + "resourceName": "authority" } ``` -* `resourceName` parameter is optional and equal to `instance` by default. Possible values: `instance`, `authority`, `location`, +* `resourceName` parameter is required. Possible values: `authority`, `location`, `linked-data-instance`, `linked-data-work`, `linked-data-hub`. Please note that `location` reindex is synchronous. + Refer to [Indexing of Instance Records](#indexing-of-instance-records) section for reindexing of instances * `recreateIndex` parameter is optional and equal to `false` by default. If it is equal to `true` then mod-search will drop existing indices for tenant and resource, creating them again. Executing request with this parameter equal to `true` in query will erase all the tenant data in mod-search. @@ -341,13 +342,13 @@ There is no end-to-end monitoring implemented yet, however it is possible to mon how many records published to Kafka topic use inventory API: ```http -GET [OKAPI_URL]/instance-storage/reindex/[reindex job id] +GET [OKAPI_URL]/authority-storage/reindex/[reindex job id] ``` _reindex job id_ - id returned by `/search/index/inventory/reindex` endpoint. In order to estimate total records that actually added to the index, you can send a "match all" search query and check -`totalRecords`, e.g. `GET /search/instances?query=id="*"`. Alternatively you can query Elasticsearch directly, +`totalRecords`, e.g. `GET /search/authorities?query=id="*"`. Alternatively you can query Elasticsearch directly, see [ES search API](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-match-all-query.html#query-dsl-match-all-query). ## Indexing of Instance Records diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index 5acfe5540..9d3478bbe 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -4,7 +4,7 @@ "provides": [ { "id": "indices", - "version": "0.8", + "version": "1.0", "handlers": [ { "methods": [ diff --git a/src/main/java/org/folio/search/controller/IndexManagementController.java b/src/main/java/org/folio/search/controller/IndexManagementController.java index b700444e2..146e06fcd 100644 --- a/src/main/java/org/folio/search/controller/IndexManagementController.java +++ b/src/main/java/org/folio/search/controller/IndexManagementController.java @@ -6,6 +6,7 @@ import org.folio.search.domain.dto.CreateIndexRequest; import org.folio.search.domain.dto.FolioCreateIndexResponse; import org.folio.search.domain.dto.FolioIndexOperationResponse; +import org.folio.search.domain.dto.IndexSettings; import org.folio.search.domain.dto.ReindexJob; import org.folio.search.domain.dto.ReindexRequest; import org.folio.search.domain.dto.ReindexStatusItem; @@ -50,15 +51,15 @@ public ResponseEntity indexRecords(List reindexInstanceRecords(String tenantId) { + public ResponseEntity reindexInstanceRecords(String tenantId, IndexSettings indexSettings) { log.info("Attempting to run full-reindex for instance records [tenant: {}]", tenantId); - reindexService.submitFullReindex(tenantId); + reindexService.submitFullReindex(tenantId, indexSettings); return ResponseEntity.ok().build(); } @Override public ResponseEntity reindexUploadInstanceRecords(String tenantId, ReindexUploadDto reindexUploadDto) { - reindexService.submitUploadReindex(tenantId, reindexUploadDto.getEntityTypes()); + reindexService.submitUploadReindex(tenantId, reindexUploadDto); return ResponseEntity.ok().build(); } diff --git a/src/main/java/org/folio/search/service/IndexService.java b/src/main/java/org/folio/search/service/IndexService.java index 51bebe479..dc440d9b9 100644 --- a/src/main/java/org/folio/search/service/IndexService.java +++ b/src/main/java/org/folio/search/service/IndexService.java @@ -271,9 +271,7 @@ private void updateRefreshInterval(ObjectNode settings, Integer refreshInt) { } private static ResourceType getReindexRequestResourceType(ReindexRequest req) { - return req == null || req.getResourceName() == null - ? ResourceType.INSTANCE - : ResourceType.byName(req.getResourceName().getValue()); + return ResourceType.byName(req.getResourceName().getValue()); } private void validateResourceName(ResourceType resourceName, String message) { diff --git a/src/main/java/org/folio/search/service/SearchTenantService.java b/src/main/java/org/folio/search/service/SearchTenantService.java index 45404f05b..682176182 100644 --- a/src/main/java/org/folio/search/service/SearchTenantService.java +++ b/src/main/java/org/folio/search/service/SearchTenantService.java @@ -9,9 +9,11 @@ import org.folio.search.configuration.properties.SearchConfigurationProperties; import org.folio.search.domain.dto.LanguageConfig; import org.folio.search.domain.dto.ReindexRequest; +import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.browse.CallNumberBrowseRangeService; import org.folio.search.service.consortium.LanguageConfigServiceDecorator; import org.folio.search.service.metadata.ResourceDescriptionService; +import org.folio.search.service.reindex.ReindexService; import org.folio.spring.FolioExecutionContext; import org.folio.spring.liquibase.FolioSpringLiquibase; import org.folio.spring.service.PrepareSystemUserService; @@ -32,6 +34,7 @@ public class SearchTenantService extends TenantService { private static final String CENTRAL_TENANT_ID_PARAM_NAME = "centralTenantId"; private final IndexService indexService; + private final ReindexService reindexService; private final KafkaAdminService kafkaAdminService; private final PrepareSystemUserService prepareSystemUserService; private final LanguageConfigServiceDecorator languageConfigService; @@ -41,7 +44,7 @@ public class SearchTenantService extends TenantService { public SearchTenantService(JdbcTemplate jdbcTemplate, FolioExecutionContext context, FolioSpringLiquibase folioSpringLiquibase, KafkaAdminService kafkaAdminService, - IndexService indexService, + IndexService indexService, ReindexService reindexService, PrepareSystemUserService prepareSystemUserService, LanguageConfigServiceDecorator languageConfigService, CallNumberBrowseRangeService callNumberBrowseRangeService, @@ -50,6 +53,7 @@ public SearchTenantService(JdbcTemplate jdbcTemplate, FolioExecutionContext cont super(jdbcTemplate, context, folioSpringLiquibase); this.kafkaAdminService = kafkaAdminService; this.indexService = indexService; + this.reindexService = reindexService; this.prepareSystemUserService = prepareSystemUserService; this.languageConfigService = languageConfigService; this.callNumberBrowseRangeService = callNumberBrowseRangeService; @@ -167,7 +171,12 @@ private void createIndexesAndReindex(TenantAttributes tenantAttributes) { .filter(parameter -> parameter.getKey().equals(REINDEX_PARAM_NAME) && parseBoolean(parameter.getValue())) .findFirst() .ifPresent(parameter -> resourceNames.forEach(resource -> { - if (resourceDescriptionService.get(resource).isReindexSupported()) { + if (!resourceDescriptionService.get(resource).isReindexSupported()) { + return; + } + if (resource.getName().equals(ReindexEntityType.INSTANCE.getType())) { + reindexService.submitFullReindex(context.getTenantId(), null); + } else { indexService.reindexInventory(context.getTenantId(), new ReindexRequest().resourceName(ReindexRequest.ResourceNameEnum.fromValue(resource.getName()))); } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java b/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java index 6701e863c..ec66b8d77 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexCommonService.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.stream.Collectors; import lombok.extern.log4j.Log4j2; +import org.folio.search.domain.dto.IndexSettings; import org.folio.search.model.types.ReindexEntityType; import org.folio.search.service.IndexService; import org.folio.search.service.reindex.jdbc.ReindexJdbcRepository; @@ -33,11 +34,11 @@ public void deleteAllRecords() { } } - public void recreateIndex(ReindexEntityType reindexEntityType, String tenantId) { + public void recreateIndex(ReindexEntityType reindexEntityType, String tenantId, IndexSettings indexSettings) { try { var resourceType = RESOURCE_NAME_MAP.get(reindexEntityType); indexService.dropIndex(resourceType, tenantId); - indexService.createIndex(resourceType, tenantId); + indexService.createIndex(resourceType, tenantId, indexSettings); } catch (Exception e) { log.warn("Index cannot be recreated for resource={}, message={}", reindexEntityType, e.getMessage()); } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java index 0a8eddbb5..485124b97 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexOrchestrationService.java @@ -1,15 +1,14 @@ package org.folio.search.service.reindex; -import java.util.Arrays; import java.util.Collection; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.apache.logging.log4j.message.FormattedMessage; import org.folio.search.domain.dto.FolioIndexOperationResponse; -import org.folio.search.domain.dto.ReindexUploadDto; import org.folio.search.exception.ReindexException; import org.folio.search.model.event.ReindexRangeIndexEvent; import org.folio.search.model.event.ReindexRecordsEvent; +import org.folio.search.model.types.ReindexEntityType; import org.folio.search.repository.PrimaryResourceRepository; import org.folio.search.service.converter.MultiTenantSearchDocumentConverter; import org.folio.spring.FolioExecutionContext; @@ -64,8 +63,7 @@ public boolean process(ReindexRecordsEvent event) { event.getRangeId(), event.getRecordType()); mergeRangeService.updateFinishDate(entityType, event.getRangeId()); if (reindexStatusService.isMergeCompleted()) { - reindexService.submitUploadReindex(context.getTenantId(), - Arrays.asList(ReindexUploadDto.EntityTypesEnum.values())); + reindexService.submitUploadReindex(context.getTenantId(), ReindexEntityType.supportUploadTypes()); } } diff --git a/src/main/java/org/folio/search/service/reindex/ReindexService.java b/src/main/java/org/folio/search/service/reindex/ReindexService.java index 3dbf1b806..cf2ab1e79 100644 --- a/src/main/java/org/folio/search/service/reindex/ReindexService.java +++ b/src/main/java/org/folio/search/service/reindex/ReindexService.java @@ -11,6 +11,7 @@ import lombok.extern.log4j.Log4j2; import org.apache.commons.collections4.CollectionUtils; import org.folio.search.converter.ReindexEntityTypeMapper; +import org.folio.search.domain.dto.IndexSettings; import org.folio.search.domain.dto.ReindexUploadDto; import org.folio.search.exception.RequestValidationException; import org.folio.search.integration.folio.InventoryService; @@ -62,7 +63,7 @@ public ReindexService(ConsortiumTenantService consortiumService, this.reindexCommonService = reindexCommonService; } - public CompletableFuture submitFullReindex(String tenantId) { + public CompletableFuture submitFullReindex(String tenantId, IndexSettings indexSettings) { log.info("submitFullReindex:: for [tenantId: {}]", tenantId); validateTenant(tenantId); @@ -81,6 +82,7 @@ public CompletableFuture submitFullReindex(String tenantId) { mergeRangeService.saveMergeRanges(rangesForAllTenants); }, reindexFullExecutor) .thenRun(() -> publishRecordsRange(tenantId)) + .thenRun(() -> recreateIndices(tenantId, ReindexEntityType.supportUploadTypes(), indexSettings)) .handle((unused, throwable) -> { if (throwable != null) { log.error("initFullReindex:: process failed [tenantId: {}, error: {}]", tenantId, throwable); @@ -94,20 +96,34 @@ public CompletableFuture submitFullReindex(String tenantId) { } public CompletableFuture submitUploadReindex(String tenantId, - List entityTypes) { - log.info("submitUploadReindex:: for [tenantId: {}, entities: {}]", tenantId, entityTypes); - var reindexEntityTypes = entityTypeMapper.convert(entityTypes) + ReindexUploadDto reindexUploadDto) { + var entityTypes = entityTypeMapper.convert(reindexUploadDto.getEntityTypes()) .stream().filter(ReindexEntityType::isSupportsUpload).toList(); + return submitUploadReindex(tenantId, entityTypes, true, reindexUploadDto.getIndexSettings()); + } + + public CompletableFuture submitUploadReindex(String tenantId, + List entityTypes) { + return submitUploadReindex(tenantId, entityTypes, false, null); + } - validateUploadReindex(tenantId, reindexEntityTypes); + private CompletableFuture submitUploadReindex(String tenantId, + List entityTypes, + boolean recreateIndex, + IndexSettings indexSettings) { + log.info("submitUploadReindex:: for [tenantId: {}, entities: {}]", tenantId, entityTypes); - for (var reindexEntityType : reindexEntityTypes) { + validateUploadReindex(tenantId, entityTypes); + + for (var reindexEntityType : entityTypes) { statusService.recreateUploadStatusRecord(reindexEntityType); - reindexCommonService.recreateIndex(reindexEntityType, tenantId); + if (recreateIndex) { + reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings); + } } var futures = new ArrayList<>(); - for (var entityType : reindexEntityTypes) { + for (var entityType : entityTypes) { var future = CompletableFuture.runAsync(() -> uploadRangeService.prepareAndSendIndexRanges(entityType), reindexUploadExecutor) .handle((unused, throwable) -> { @@ -124,6 +140,12 @@ public CompletableFuture submitUploadReindex(String tenantId, return CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])); } + private void recreateIndices(String tenantId, List entityTypes, IndexSettings indexSettings) { + for (var reindexEntityType : entityTypes) { + reindexCommonService.recreateIndex(reindexEntityType, tenantId, indexSettings); + } + } + private List processForConsortium(String tenantId) { List mergeRangeEntities = new ArrayList<>(); var memberTenants = consortiumService.getConsortiumTenants(tenantId); diff --git a/src/main/resources/swagger.api/examples/request/indexSettings.yaml b/src/main/resources/swagger.api/examples/request/indexSettings.yaml new file mode 100644 index 000000000..9184ec0c1 --- /dev/null +++ b/src/main/resources/swagger.api/examples/request/indexSettings.yaml @@ -0,0 +1,4 @@ +value: + numberOfShards: 1 + numberOfReplicas: 1 + refreshInterval: 1 \ No newline at end of file diff --git a/src/main/resources/swagger.api/examples/request/reindexRequest.yaml b/src/main/resources/swagger.api/examples/request/reindexRequest.yaml index 45cea543a..6576f53ce 100644 --- a/src/main/resources/swagger.api/examples/request/reindexRequest.yaml +++ b/src/main/resources/swagger.api/examples/request/reindexRequest.yaml @@ -1,3 +1,3 @@ value: recreateIndex: true - resourceName: instance \ No newline at end of file + resourceName: authority \ No newline at end of file diff --git a/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml b/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml index 0cb31204a..dff50dd52 100644 --- a/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml +++ b/src/main/resources/swagger.api/paths/reindex-instance-records/reindex-instance-records-full.yaml @@ -4,6 +4,14 @@ post: description: Initiates the full reindex for the inventory instance records tags: - index-management + requestBody: + content: + application/json: + examples: + indexSettings: + $ref: '../../examples/request/indexSettings.yaml' + schema: + $ref: '../../schemas/entity/indexSettings.yaml' parameters: - $ref: '../../parameters/x-okapi-tenant-header.yaml' responses: diff --git a/src/main/resources/swagger.api/paths/search-index/search-index-inventory-reindex.yaml b/src/main/resources/swagger.api/paths/search-index/search-index-inventory-reindex.yaml index b52820e07..ed0b356c9 100644 --- a/src/main/resources/swagger.api/paths/search-index/search-index-inventory-reindex.yaml +++ b/src/main/resources/swagger.api/paths/search-index/search-index-inventory-reindex.yaml @@ -5,6 +5,7 @@ post: tags: - index-management requestBody: + required: true content: application/json: examples: diff --git a/src/main/resources/swagger.api/schemas/request/reindexRequest.yaml b/src/main/resources/swagger.api/schemas/request/reindexRequest.yaml index 9795f84d1..c4589915a 100644 --- a/src/main/resources/swagger.api/schemas/request/reindexRequest.yaml +++ b/src/main/resources/swagger.api/schemas/request/reindexRequest.yaml @@ -8,9 +8,7 @@ properties: resourceName: type: string description: Resource name to run reindex for - default: instance enum: - - instance - authority - location - linked-data-instance @@ -19,3 +17,5 @@ properties: indexSettings: description: Index settings to apply for index $ref: "../../schemas/entity/indexSettings.yaml" +required: + - resourceName diff --git a/src/main/resources/swagger.api/schemas/request/reindexUploadDto.yaml b/src/main/resources/swagger.api/schemas/request/reindexUploadDto.yaml index 639257667..24c62e83b 100644 --- a/src/main/resources/swagger.api/schemas/request/reindexUploadDto.yaml +++ b/src/main/resources/swagger.api/schemas/request/reindexUploadDto.yaml @@ -11,5 +11,8 @@ properties: - contributor - classification minItems: 1 + indexSettings: + description: Index settings to apply for index + $ref: '../entity/indexSettings.yaml' required: - entityTypes diff --git a/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java b/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java index 6c5ac3474..e9e92b18d 100644 --- a/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java +++ b/src/test/java/org/folio/search/controller/IndexManagementControllerTest.java @@ -11,6 +11,7 @@ import static org.folio.search.utils.TestUtils.mapOf; import static org.folio.search.utils.TestUtils.randomId; import static org.folio.search.utils.TestUtils.resourceEvent; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.eq; @@ -31,6 +32,7 @@ import java.util.concurrent.CompletableFuture; import org.folio.search.domain.dto.CreateIndexRequest; import org.folio.search.domain.dto.IndexDynamicSettings; +import org.folio.search.domain.dto.IndexSettings; import org.folio.search.domain.dto.ReindexJob; import org.folio.search.domain.dto.ReindexRequest; import org.folio.search.domain.dto.ReindexStatusItem; @@ -80,9 +82,19 @@ class IndexManagementControllerTest { @Test void submitReindexFull_positive() throws Exception { - when(reindexService.submitFullReindex(TENANT_ID)).thenReturn(new CompletableFuture<>()); + when(reindexService.submitFullReindex(TENANT_ID, null)).thenReturn(new CompletableFuture<>()); - mockMvc.perform(post(reindexFullPath()).header(XOkapiHeaders.TENANT, TENANT_ID)) + mockMvc.perform(post(reindexFullPath()) + .contentType(APPLICATION_JSON).header(XOkapiHeaders.TENANT, TENANT_ID)) + .andExpect(status().isOk()); + } + + @Test + void submitReindexFull_positive_withSettings() throws Exception { + var requestBody = new IndexSettings().numberOfShards(1).refreshInterval(2).numberOfReplicas(3); + when(reindexService.submitFullReindex(TENANT_ID, requestBody)).thenReturn(new CompletableFuture<>()); + + mockMvc.perform(preparePostRequest(reindexFullPath(), asJsonString(requestBody))) .andExpect(status().isOk()); } @@ -96,6 +108,19 @@ void submitReindexUpload_positive() throws Exception { .andExpect(status().isOk()); } + @Test + void submitReindexUpload_positive_withSettings() throws Exception { + var indexSettings = new IndexSettings().numberOfShards(1).refreshInterval(2).numberOfReplicas(3); + var requestBody = new ReindexUploadDto() + .addEntityTypesItem(ReindexUploadDto.EntityTypesEnum.INSTANCE) + .indexSettings(indexSettings); + when(reindexService.submitUploadReindex(TENANT_ID, requestBody)).thenReturn(new CompletableFuture<>()); + + mockMvc.perform(preparePostRequest(reindexUploadPath(), asJsonString(requestBody)) + .header(XOkapiHeaders.TENANT, TENANT_ID)) + .andExpect(status().isOk()); + } + @Test void createIndex_positive() throws Exception { when(indexService.createIndex(RESOURCE, TENANT_ID)) @@ -211,10 +236,12 @@ void updateIndexSettings_positive() throws Exception { @Test void canSubmitReindex() throws Exception { var jobId = randomId(); - when(indexService.reindexInventory(TENANT_ID, null)).thenReturn(new ReindexJob().id(jobId)); + var request = new ReindexRequest().resourceName(ReindexRequest.ResourceNameEnum.AUTHORITY); + when(indexService.reindexInventory(TENANT_ID, request)).thenReturn(new ReindexJob().id(jobId)); mockMvc.perform(post("/search/index/inventory/reindex") .header(XOkapiHeaders.TENANT, TENANT_ID) + .content(asJsonString(request)) .contentType(APPLICATION_JSON)) .andExpect(status().isOk()) .andExpect(jsonPath("id", is(jobId))); @@ -223,7 +250,7 @@ void canSubmitReindex() throws Exception { @Test void reindexInventoryRecords_positive_withRecreateIndexFlag() throws Exception { var jobId = randomId(); - var request = new ReindexRequest().recreateIndex(true); + var request = new ReindexRequest().resourceName(ReindexRequest.ResourceNameEnum.AUTHORITY).recreateIndex(true); when(indexService.reindexInventory(TENANT_ID, request)).thenReturn(new ReindexJob().id(jobId)); mockMvc.perform(post("/search/index/inventory/reindex") @@ -235,6 +262,17 @@ void reindexInventoryRecords_positive_withRecreateIndexFlag() throws Exception { .andExpect(jsonPath("id", is(jobId))); } + @Test + void canSubmitReindex_negative_noBody() throws Exception { + var jobId = randomId(); + when(indexService.reindexInventory(TENANT_ID, null)).thenReturn(new ReindexJob().id(jobId)); + + mockMvc.perform(post("/search/index/inventory/reindex") + .header(XOkapiHeaders.TENANT, TENANT_ID) + .contentType(APPLICATION_JSON)) + .andExpect(status().isBadRequest()); + } + @Test void reindexInventoryRecords_negative_emptyContentTypeAndBody() throws Exception { var jobId = randomId(); @@ -253,14 +291,16 @@ void reindexInventoryRecords_negative_emptyContentTypeAndBody() throws Exception @Test void reindexInventoryRecords_negative_constraintViolationException() throws Exception { + var request = new ReindexRequest().resourceName(ReindexRequest.ResourceNameEnum.AUTHORITY); var constraintViolation = mock(ConstraintViolationImpl.class); when(constraintViolation.getPropertyPath()).thenReturn(PathImpl.createPathFromString("recreateIndices")); when(constraintViolation.getMessage()).thenReturn("must be boolean"); - when(indexService.reindexInventory(TENANT_ID, null)).thenThrow( + when(indexService.reindexInventory(TENANT_ID, request)).thenThrow( new ConstraintViolationException("error", Set.>of(constraintViolation))); mockMvc.perform(post("/search/index/inventory/reindex") .header(XOkapiHeaders.TENANT, TENANT_ID) + .content(asJsonString(request)) .contentType(APPLICATION_JSON)) .andExpect(status().isBadRequest()) .andExpect(jsonPath("$.total_records", is(1))) @@ -271,14 +311,16 @@ void reindexInventoryRecords_negative_constraintViolationException() throws Exce @Test void reindexInventoryRecords_negative_illegalArgumentException() throws Exception { - when(indexService.reindexInventory(TENANT_ID, null)).thenThrow(new IllegalArgumentException("invalid value")); + var request = new ReindexRequest().resourceName(ReindexRequest.ResourceNameEnum.AUTHORITY); + when(indexService.reindexInventory(TENANT_ID, request)).thenThrow(new IllegalArgumentException("invalid value")); mockMvc.perform(post("/search/index/inventory/reindex") .header(XOkapiHeaders.TENANT, TENANT_ID) + .content(asJsonString(request)) .contentType(APPLICATION_JSON)) .andExpect(status().isBadRequest()) .andExpect(jsonPath("$.total_records", is(1))) - .andExpect(jsonPath("$.errors[0].message", is("invalid value"))) + .andExpect(jsonPath("$.errors[0].message", containsString("invalid value"))) .andExpect(jsonPath("$.errors[0].type", is("IllegalArgumentException"))) .andExpect(jsonPath("$.errors[0].code", is("validation_error"))); } diff --git a/src/test/java/org/folio/search/controller/IndexManagementIT.java b/src/test/java/org/folio/search/controller/IndexManagementIT.java index dae592306..aacb3dd36 100644 --- a/src/test/java/org/folio/search/controller/IndexManagementIT.java +++ b/src/test/java/org/folio/search/controller/IndexManagementIT.java @@ -65,23 +65,10 @@ static void cleanUp() { removeTenant(); } - @Test - void runReindex_positive_instance() throws Exception { - var request = post(ApiEndpoints.reindexPath()) - .headers(defaultHeaders()) - .header(XOkapiHeaders.URL, okapi.getOkapiUrl()) - .contentType(MediaType.APPLICATION_JSON); - - mockMvc.perform(request) - .andExpect(status().isOk()) - .andExpect(jsonPath("$.id", is("77ef33c0-2774-45e9-9f45-eb54082e2820"))) - .andExpect(jsonPath("$.jobStatus", is("In progress"))) - .andExpect(jsonPath("$.submittedDate", is("2021-11-08T12:00:00.000+00:00"))); - } - @Test void runReindex_positive_authority() throws Exception { - var request = getReindexRequestBuilder(asJsonString(new ReindexRequest().resourceName(AUTHORITY))); + var request = getReindexRequestBuilder( + asJsonString(new ReindexRequest().resourceName(AUTHORITY))); mockMvc.perform(request) .andExpect(status().isOk()) diff --git a/src/test/java/org/folio/search/service/IndexServiceTest.java b/src/test/java/org/folio/search/service/IndexServiceTest.java index fbffc0a9f..958231daa 100644 --- a/src/test/java/org/folio/search/service/IndexServiceTest.java +++ b/src/test/java/org/folio/search/service/IndexServiceTest.java @@ -7,7 +7,6 @@ import static org.folio.search.model.types.ResourceType.AUTHORITY; import static org.folio.search.model.types.ResourceType.CAMPUS; import static org.folio.search.model.types.ResourceType.INSTANCE; -import static org.folio.search.model.types.ResourceType.INSTANCE_SUBJECT; import static org.folio.search.model.types.ResourceType.INSTITUTION; import static org.folio.search.model.types.ResourceType.LIBRARY; import static org.folio.search.model.types.ResourceType.LOCATION; @@ -23,12 +22,10 @@ import static org.folio.search.utils.TestConstants.TENANT_ID; import static org.folio.search.utils.TestUtils.randomId; import static org.folio.search.utils.TestUtils.resourceDescription; -import static org.folio.search.utils.TestUtils.secondaryResourceDescription; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.atMostOnce; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.eq; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.times; @@ -254,20 +251,21 @@ void createIndexIfNotExist_shouldNotCreateIndex_alreadyExist() { @Test void reindexInventory_positive_recreateIndexIsTrue() { - var indexName = getIndexName(INSTANCE, TENANT_ID); + var indexName = getIndexName(AUTHORITY, TENANT_ID); var createIndexResponse = getSuccessFolioCreateIndexResponse(List.of(indexName)); var expectedResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://instance-storage/reindex"); + var expectedUri = URI.create("http://authority-storage/reindex"); when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - when(mappingsHelper.getMappings(INSTANCE)).thenReturn(EMPTY_OBJECT); - when(settingsHelper.getSettingsJson(INSTANCE)).thenReturn(EMPTY_JSON_OBJECT); + when(mappingsHelper.getMappings(AUTHORITY)).thenReturn(EMPTY_OBJECT); + when(settingsHelper.getSettingsJson(AUTHORITY)).thenReturn(EMPTY_JSON_OBJECT); when(indexRepository.indexExists(indexName)).thenReturn(true, false); when(indexRepository.createIndex(indexName, EMPTY_OBJECT, EMPTY_OBJECT)).thenReturn(createIndexResponse); - when(resourceDescriptionService.find(INSTANCE)).thenReturn( - Optional.of(resourceDescription(INSTANCE))); + when(resourceDescriptionService.find(AUTHORITY)).thenReturn( + Optional.of(resourceDescription(AUTHORITY))); - var actual = indexService.reindexInventory(TENANT_ID, new ReindexRequest().recreateIndex(true)); + var actual = indexService.reindexInventory(TENANT_ID, + new ReindexRequest().resourceName(ResourceNameEnum.AUTHORITY).recreateIndex(true)); assertThat(actual).isEqualTo(expectedResponse); verify(indexRepository).dropIndex(indexName); @@ -277,14 +275,15 @@ void reindexInventory_positive_recreateIndexIsTrue() { @Test void reindexInventory_positive_recreateIndexIsTrue_memberTenant() { var expectedResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://instance-storage/reindex"); + var expectedUri = URI.create("http://authority-storage/reindex"); when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - when(resourceDescriptionService.find(INSTANCE)).thenReturn( - Optional.of(resourceDescription(INSTANCE))); + when(resourceDescriptionService.find(AUTHORITY)).thenReturn( + Optional.of(resourceDescription(AUTHORITY))); when(tenantProvider.getTenant(TENANT_ID)).thenReturn(CENTRAL_TENANT_ID); - var actual = indexService.reindexInventory(TENANT_ID, new ReindexRequest().recreateIndex(true)); + var actual = indexService.reindexInventory(TENANT_ID, + new ReindexRequest().resourceName(ResourceNameEnum.AUTHORITY).recreateIndex(true)); assertThat(actual).isEqualTo(expectedResponse); verifyNoInteractions(indexRepository); @@ -293,114 +292,16 @@ void reindexInventory_positive_recreateIndexIsTrue_memberTenant() { @Test void reindexInventory_positive_recreateIndexIsFalse() { - var expectedResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://instance-storage/reindex"); - - when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - when(resourceDescriptionService.find(INSTANCE)).thenReturn( - Optional.of(resourceDescription(INSTANCE))); - - var actual = indexService.reindexInventory(TENANT_ID, new ReindexRequest()); - assertThat(actual).isEqualTo(expectedResponse); - verifyNoInteractions(locationService); - } - - @Test - void reindexInventory_positive_resourceNameIsNull() { - var expectedResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://instance-storage/reindex"); - - when(resourceDescriptionService.find(INSTANCE)).thenReturn( - Optional.of(resourceDescription(INSTANCE))); - when(resourceDescriptionService.getSecondaryResourceTypes(INSTANCE)).thenReturn( - List.of(UNKNOWN)); - when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - - var actual = indexService.reindexInventory(TENANT_ID, new ReindexRequest().resourceName(null)); - assertThat(actual).isEqualTo(expectedResponse); - verifyNoInteractions(locationService); - } - - @Test - void reindexInventory_positive_resourceNameIsNullAndRecreateIndexIsTrue() { - var expectedResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://instance-storage/reindex"); - - when(resourceDescriptionService.find(INSTANCE)).thenReturn( - Optional.of(resourceDescription(INSTANCE))); - when(resourceDescriptionService.find(INSTANCE_SUBJECT)) - .thenReturn(Optional.of(secondaryResourceDescription(INSTANCE_SUBJECT, INSTANCE))); - when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - when(resourceDescriptionService.getSecondaryResourceTypes(INSTANCE)) - .thenReturn(List.of(INSTANCE_SUBJECT)); - mockCreateIndexOperation(INSTANCE); - mockCreateIndexOperation(INSTANCE_SUBJECT); - - var secondaryIndexName = getIndexName(INSTANCE_SUBJECT, TENANT_ID); - var instanceIndexName = getIndexName(INSTANCE, TENANT_ID); - when(indexRepository.indexExists(instanceIndexName)).thenReturn(true); - when(indexRepository.indexExists(secondaryIndexName)).thenReturn(true); - - var actual = indexService.reindexInventory(TENANT_ID, new ReindexRequest().resourceName(null).recreateIndex(true)); - assertThat(actual).isEqualTo(expectedResponse); - - verify(indexRepository).dropIndex(instanceIndexName); - verify(indexRepository).dropIndex(secondaryIndexName); - verifyNoInteractions(locationService); - } - - @Test - void reindexInventory_positive_reindexRequestIsNull() { - var expectedResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://instance-storage/reindex"); - - when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - when(resourceDescriptionService.find(INSTANCE)).thenReturn( - Optional.of(resourceDescription(INSTANCE))); - when(resourceDescriptionService.getSecondaryResourceTypes(INSTANCE)).thenReturn( - List.of(UNKNOWN)); - - var actual = indexService.reindexInventory(TENANT_ID, null); - - assertThat(actual).isEqualTo(expectedResponse); - verifyNoInteractions(locationService); - } - - @Test - void reindexInventory_positive_authorityRecord() { var expectedResponse = new ReindexJob().id(randomId()); var expectedUri = URI.create("http://authority-storage/reindex"); - var resourceName = AUTHORITY; when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(expectedResponse); - when(resourceDescriptionService.find(resourceName)) - .thenReturn(Optional.of(resourceDescription(resourceName))); - - var actual = indexService.reindexInventory(TENANT_ID, new ReindexRequest() - .resourceName(ResourceNameEnum.AUTHORITY)); - - assertThat(actual).isEqualTo(expectedResponse); - verifyNoInteractions(locationService); - } - - @Test - void reindexInventory_positive_authorityRecordAndRecreateIndex() { - var reindexResponse = new ReindexJob().id(randomId()); - var expectedUri = URI.create("http://authority-storage/reindex"); - var indexName = getIndexName(AUTHORITY, TENANT_ID); - - when(resourceReindexClient.submitReindex(expectedUri)).thenReturn(reindexResponse); when(resourceDescriptionService.find(AUTHORITY)).thenReturn( Optional.of(resourceDescription(AUTHORITY))); - when(mappingsHelper.getMappings(AUTHORITY)).thenReturn(EMPTY_OBJECT); - when(settingsHelper.getSettingsJson(AUTHORITY)).thenReturn(EMPTY_JSON_OBJECT); - when(indexRepository.createIndex(indexName, EMPTY_OBJECT, EMPTY_OBJECT)) - .thenReturn(getSuccessFolioCreateIndexResponse(List.of(indexName))); - var reindexRequest = new ReindexRequest().resourceName(ResourceNameEnum.AUTHORITY).recreateIndex(true); - var actual = indexService.reindexInventory(TENANT_ID, reindexRequest); - - assertThat(actual).isEqualTo(reindexResponse); + var actual = indexService.reindexInventory(TENANT_ID, + new ReindexRequest().resourceName(ResourceNameEnum.AUTHORITY)); + assertThat(actual).isEqualTo(expectedResponse); verifyNoInteractions(locationService); } @@ -543,14 +444,6 @@ private String getIndexDynamicSettingsJsonString(Integer replicas, String refres "refresh_interval", refresh))); } - private void mockCreateIndexOperation(ResourceType resource) { - var indexName = getIndexName(resource, TENANT_ID); - doReturn(EMPTY_OBJECT).when(mappingsHelper).getMappings(resource); - doReturn(EMPTY_JSON_OBJECT).when(settingsHelper).getSettingsJson(resource); - doReturn(getSuccessFolioCreateIndexResponse(List.of(indexName))).when(indexRepository) - .createIndex(indexName, EMPTY_OBJECT, EMPTY_OBJECT); - } - private static Stream customSettingsTestData() { return Stream.of( Arguments.of(1, 1, 2), diff --git a/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java b/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java index 50148d9a6..bad12fb62 100644 --- a/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java +++ b/src/test/java/org/folio/search/service/reindex/ReindexServiceTest.java @@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService; import org.apache.commons.lang3.ThreadUtils; import org.folio.search.converter.ReindexEntityTypeMapper; +import org.folio.search.domain.dto.IndexSettings; import org.folio.search.domain.dto.ReindexUploadDto; import org.folio.search.exception.FolioIntegrationException; import org.folio.search.exception.RequestValidationException; @@ -73,7 +74,7 @@ class ReindexServiceTest { void submitFullReindex_negative_shouldFailForEcsMemberTenant() { when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of("central")); - assertThrows(RequestValidationException.class, () -> reindexService.submitFullReindex(TENANT_ID), + assertThrows(RequestValidationException.class, () -> reindexService.submitFullReindex(TENANT_ID, null), REQUEST_NOT_ALLOWED_MSG); } @@ -96,8 +97,9 @@ void submitFullReindex_positive() throws InterruptedException { return null; }).when(reindexExecutor).execute(any()); final var expectedCallsCount = ReindexEntityType.supportMergeTypes().size(); + final var indexSettings = new IndexSettings().refreshInterval(1).numberOfShards(2).numberOfReplicas(3); - reindexService.submitFullReindex(tenant); + reindexService.submitFullReindex(tenant, indexSettings); ThreadUtils.sleep(Duration.ofSeconds(1)); verify(reindexCommonService).deleteAllRecords(); @@ -110,6 +112,8 @@ void submitFullReindex_positive() throws InterruptedException { .updateReindexMergeStarted(any(ReindexEntityType.class), eq(1)); verify(mergeRangeService, times(expectedCallsCount)).fetchMergeRanges(any(ReindexEntityType.class)); verify(mergeRangeService).truncateMergeRanges(); + verify(reindexCommonService, times(ReindexEntityType.supportUploadTypes().size())) + .recreateIndex(any(), eq(tenant), eq(indexSettings)); verifyNoMoreInteractions(mergeRangeService); } @@ -136,7 +140,7 @@ void submitFullReindex_negative_abortMergeAndSetFailedStatusWhenPublishingRanges .doThrow(FolioIntegrationException.class) .when(reindexExecutor).execute(any()); - reindexService.submitFullReindex(tenant); + reindexService.submitFullReindex(tenant, null); ThreadUtils.sleep(Duration.ofSeconds(1)); verify(mergeRangeService).saveMergeRanges(anyList()); @@ -152,7 +156,7 @@ void submitUploadReindex_negative_notAllowedToRunUploadForEcsMemberTenant() { var member = "member"; var central = "central"; when(consortiumService.getCentralTenant(member)).thenReturn(Optional.of(central)); - List entityTypes = List.of(); + List entityTypes = List.of(); // act & assert assertThrows(RequestValidationException.class, @@ -165,7 +169,7 @@ void submitUploadReindex_negative_notAllowedToRunUploadWhenMergeNotComplete() { when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of(TENANT_ID)); when(statusService.getStatusesByType()) .thenReturn(Map.of(HOLDINGS, ReindexStatus.MERGE_IN_PROGRESS)); - var entityTypes = List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE); + var entityTypes = List.of(ReindexEntityType.INSTANCE); // act & assert assertThrows(RequestValidationException.class, @@ -182,7 +186,7 @@ void submitUploadReindex_negative_notAllowedToRunUploadWhenUploadInProgress() { INSTANCE, ReindexStatus.MERGE_COMPLETED, ITEM, ReindexStatus.MERGE_IN_PROGRESS, HOLDINGS, ReindexStatus.MERGE_FAILED)); - var entityTypes = List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE); + var entityTypes = List.of(ReindexEntityType.INSTANCE); // act & assert assertThrows(RequestValidationException.class, @@ -193,13 +197,28 @@ void submitUploadReindex_negative_notAllowedToRunUploadWhenUploadInProgress() { @Test void submitUploadReindex_positive() { when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of(TENANT_ID)); - when(entityTypeMapper.convert(List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE))).thenReturn(List.of(INSTANCE)); doAnswer(invocation -> { ((Runnable) invocation.getArgument(0)).run(); return null; }).when(reindexExecutor).execute(any()); - reindexService.submitUploadReindex(TENANT_ID, List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE)); + reindexService.submitUploadReindex(TENANT_ID, List.of(ReindexEntityType.INSTANCE)); + + verify(statusService).recreateUploadStatusRecord(INSTANCE); + verify(uploadRangeService).prepareAndSendIndexRanges(INSTANCE); + } + + @Test + void submitUploadReindex_positive_recreateIndex() { + var uploadDto = new ReindexUploadDto().entityTypes(List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE)); + when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of(TENANT_ID)); + when(entityTypeMapper.convert(uploadDto.getEntityTypes())).thenReturn(List.of(INSTANCE)); + doAnswer(invocation -> { + ((Runnable) invocation.getArgument(0)).run(); + return null; + }).when(reindexExecutor).execute(any()); + + reindexService.submitUploadReindex(TENANT_ID, uploadDto); verify(statusService).recreateUploadStatusRecord(INSTANCE); verify(uploadRangeService).prepareAndSendIndexRanges(INSTANCE); @@ -208,7 +227,6 @@ void submitUploadReindex_positive() { @Test void submitUploadReindex_negative_failedToPrepareUploadRangesAndSendThem() { when(consortiumService.getCentralTenant(TENANT_ID)).thenReturn(Optional.of(TENANT_ID)); - when(entityTypeMapper.convert(List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE))).thenReturn(List.of(INSTANCE)); doThrow(RuntimeException.class).when(uploadRangeService).prepareAndSendIndexRanges(INSTANCE); doAnswer(invocation -> { @@ -216,7 +234,7 @@ void submitUploadReindex_negative_failedToPrepareUploadRangesAndSendThem() { return null; }).when(reindexExecutor).execute(any()); - reindexService.submitUploadReindex(TENANT_ID, List.of(ReindexUploadDto.EntityTypesEnum.INSTANCE)); + reindexService.submitUploadReindex(TENANT_ID, List.of(ReindexEntityType.INSTANCE)); verify(statusService).updateReindexUploadFailed(INSTANCE); } diff --git a/src/test/resources/mappings/inventory.json b/src/test/resources/mappings/inventory.json index bda19d675..063e3f4ca 100644 --- a/src/test/resources/mappings/inventory.json +++ b/src/test/resources/mappings/inventory.json @@ -188,23 +188,6 @@ } } }, - { - "request": { - "method": "POST", - "url": "/instance-storage/reindex" - }, - "response": { - "status": 200, - "jsonBody": { - "id": "77ef33c0-2774-45e9-9f45-eb54082e2820", - "jobStatus": "In progress", - "submittedDate": "2021-11-08T12:00:00.000+00:00" - }, - "headers": { - "Content-Type": "application/json" - } - } - }, { "request": { "method": "POST",