-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[MSEARCH-771]. Implement Indexing of Institutions from Kafka (#607)
- Loading branch information
1 parent
acb43ac
commit d094df4
Showing
11 changed files
with
241 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
27 changes: 27 additions & 0 deletions
27
src/main/java/org/folio/search/model/dto/locationunit/InstitutionDto.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| package org.folio.search.model.dto.locationunit; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonInclude; | ||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import lombok.Builder; | ||
| import lombok.Data; | ||
| import lombok.With; | ||
| import lombok.extern.jackson.Jacksonized; | ||
|
|
||
| /** | ||
| * Describes Institution object that comes from external channels. | ||
| */ | ||
| @Data | ||
| @With | ||
| @Builder | ||
| @Jacksonized | ||
| @JsonInclude(JsonInclude.Include.NON_EMPTY) | ||
| public class InstitutionDto { | ||
|
|
||
| @JsonProperty("id") | ||
| private String id; | ||
| @JsonProperty("name") | ||
| private String name; | ||
| @JsonProperty("code") | ||
| private String code; | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| { | ||
| "index": { | ||
| "number_of_shards": 4, | ||
| "number_of_replicas": 2, | ||
| "refresh_interval": "1s", | ||
| "codec": "best_compression", | ||
| "mapping.total_fields.limit": 1000 | ||
| }, | ||
| "analysis": { | ||
| "normalizer": { | ||
| "keyword_lowercase": { | ||
| "filter": [ | ||
| "lowercase", | ||
| "icu_folding" | ||
| ], | ||
| "type": "custom" | ||
| } | ||
| }, | ||
| "tokenizers": {} | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| { | ||
| "name": "institution", | ||
| "eventBodyJavaClass": "org.folio.search.model.dto.locationunit.InstitutionDto", | ||
| "reindexSupported": false, | ||
| "fields": { | ||
| "id": { | ||
| "index": "keyword_lowercase", | ||
| "showInResponse": [ "search" ] | ||
| }, | ||
| "tenantId": { | ||
| "index": "keyword_lowercase", | ||
| "showInResponse": [ "search" ], | ||
| "isTenant": true | ||
| }, | ||
| "name": { | ||
| "index": "keyword_lowercase", | ||
| "showInResponse": [ "search" ] | ||
| }, | ||
| "code": { | ||
| "index": "keyword_lowercase", | ||
| "showInResponse": [ "search" ] | ||
| } | ||
| } | ||
| } |
144 changes: 144 additions & 0 deletions
144
src/test/java/org/folio/search/controller/InstitutionsIndexingConsortiumIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| package org.folio.search.controller; | ||
|
|
||
| import static org.assertj.core.api.Assertions.assertThat; | ||
| import static org.awaitility.Awaitility.await; | ||
| import static org.awaitility.Durations.ONE_MINUTE; | ||
| import static org.awaitility.Durations.ONE_SECOND; | ||
| import static org.folio.search.domain.dto.ResourceEventType.CREATE; | ||
| import static org.folio.search.domain.dto.ResourceEventType.DELETE; | ||
| import static org.folio.search.domain.dto.ResourceEventType.DELETE_ALL; | ||
| import static org.folio.search.domain.dto.ResourceEventType.UPDATE; | ||
| import static org.folio.search.utils.SearchUtils.INSTITUTION_RESOURCE; | ||
| import static org.folio.search.utils.SearchUtils.getIndexName; | ||
| import static org.folio.search.utils.TestConstants.CENTRAL_TENANT_ID; | ||
| import static org.folio.search.utils.TestConstants.MEMBER_TENANT_ID; | ||
| import static org.folio.search.utils.TestConstants.inventoryInstitutionTopic; | ||
| import static org.folio.search.utils.TestUtils.kafkaResourceEvent; | ||
| import static org.folio.search.utils.TestUtils.randomId; | ||
| import static org.folio.search.utils.TestUtils.toMap; | ||
| import static org.opensearch.index.query.QueryBuilders.boolQuery; | ||
| import static org.opensearch.search.builder.SearchSourceBuilder.searchSource; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Objects; | ||
| import lombok.extern.log4j.Log4j2; | ||
| import org.folio.search.domain.dto.ResourceEvent; | ||
| import org.folio.search.model.dto.locationunit.InstitutionDto; | ||
| import org.folio.search.support.base.BaseConsortiumIntegrationTest; | ||
| import org.folio.spring.testing.type.IntegrationTest; | ||
| import org.junit.jupiter.api.AfterAll; | ||
| import org.junit.jupiter.api.AfterEach; | ||
| import org.junit.jupiter.api.BeforeAll; | ||
| import org.junit.jupiter.api.Test; | ||
| import org.opensearch.action.search.SearchRequest; | ||
| import org.opensearch.client.RequestOptions; | ||
| import org.opensearch.index.query.QueryBuilders; | ||
|
|
||
| @Log4j2 | ||
| @IntegrationTest | ||
| class InstitutionsIndexingConsortiumIT extends BaseConsortiumIntegrationTest { | ||
|
|
||
| @BeforeAll | ||
| static void prepare() { | ||
| setUpTenant(CENTRAL_TENANT_ID); | ||
| setUpTenant(MEMBER_TENANT_ID); | ||
| } | ||
|
|
||
| @AfterAll | ||
| static void cleanUp() { | ||
| removeTenant(CENTRAL_TENANT_ID); | ||
| removeTenant(MEMBER_TENANT_ID); | ||
| } | ||
|
|
||
| @AfterEach | ||
| void tearDown() throws IOException { | ||
| cleanUpIndex(INSTITUTION_RESOURCE, CENTRAL_TENANT_ID); | ||
| } | ||
|
|
||
| @Test | ||
| void shouldIndexAndRemoveInstitution() { | ||
| var institution = institution(); | ||
| var createEvent = kafkaResourceEvent(CENTRAL_TENANT_ID, CREATE, toMap(institution), null); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), createEvent); | ||
|
|
||
| awaitAssertInstitutionCount(1); | ||
|
|
||
| var deleteEvent = kafkaResourceEvent(CENTRAL_TENANT_ID, DELETE, null, toMap(institution)); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), deleteEvent); | ||
|
|
||
| awaitAssertInstitutionCount(0); | ||
| } | ||
|
|
||
| @Test | ||
| void shouldIndexAndUpdateInstitution() { | ||
| var institution = institution(); | ||
| var createEvent = kafkaResourceEvent(CENTRAL_TENANT_ID, CREATE, toMap(institution), null); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), createEvent); | ||
|
|
||
| awaitAssertInstitutionCount(1); | ||
|
|
||
| var institutionUpdated = institution.withName("nameUpdated"); | ||
| var updateEvent = kafkaResourceEvent(CENTRAL_TENANT_ID, UPDATE, toMap(institutionUpdated), toMap(institution)); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), updateEvent); | ||
|
|
||
| awaitAssertInstitutionCountAfterUpdate(1, institutionUpdated); | ||
| } | ||
|
|
||
| @Test | ||
| void shouldIndexSameInstitutionFromDifferentTenantsAsSeparateDocs() { | ||
| var institution = institution(); | ||
| var createCentralEvent = kafkaResourceEvent(CENTRAL_TENANT_ID, CREATE, toMap(institution), null); | ||
| var createMemberEvent = kafkaResourceEvent(MEMBER_TENANT_ID, CREATE, toMap(institution), null); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), createCentralEvent); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), createMemberEvent); | ||
|
|
||
| awaitAssertInstitutionCount(2); | ||
| } | ||
|
|
||
| @Test | ||
| void shouldRemoveAllDocumentsByTenantIdOnDeleteAllEvent() { | ||
| var institution = institution(); | ||
| var createCentralEvent = kafkaResourceEvent(CENTRAL_TENANT_ID, CREATE, toMap(institution), null); | ||
| var createMemberEvent = kafkaResourceEvent(MEMBER_TENANT_ID, CREATE, toMap(institution), null); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), createCentralEvent); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(CENTRAL_TENANT_ID), createMemberEvent); | ||
|
|
||
| awaitAssertInstitutionCount(2); | ||
|
|
||
| var deleteAllMemberEvent = new ResourceEvent().type(DELETE_ALL).tenant(MEMBER_TENANT_ID); | ||
| kafkaTemplate.send(inventoryInstitutionTopic(MEMBER_TENANT_ID), deleteAllMemberEvent); | ||
|
|
||
| awaitAssertInstitutionCount(1); | ||
| } | ||
|
|
||
| private static InstitutionDto institution() { | ||
| return InstitutionDto.builder().id(randomId()) | ||
| .name("name") | ||
| .code("code") | ||
| .build(); | ||
| } | ||
|
|
||
| public static void awaitAssertInstitutionCount(int expected) { | ||
| await().atMost(ONE_MINUTE).pollInterval(ONE_SECOND).untilAsserted(() -> { | ||
| var totalHits = countIndexDocument(INSTITUTION_RESOURCE, CENTRAL_TENANT_ID); | ||
|
|
||
| assertThat(totalHits).isEqualTo(expected); | ||
| }); | ||
| } | ||
|
|
||
| public static void awaitAssertInstitutionCountAfterUpdate(int expected, InstitutionDto institutionUpdated) { | ||
| await().atMost(ONE_MINUTE).pollInterval(ONE_SECOND).untilAsserted(() -> { | ||
| var idQuery = QueryBuilders.matchQuery("id", institutionUpdated.getId()); | ||
| var nameQuery = QueryBuilders.matchQuery("name", institutionUpdated.getName()); | ||
|
|
||
| var searchRequest = new SearchRequest() | ||
| .source(searchSource().query(boolQuery().must(idQuery).must(nameQuery)) | ||
| .trackTotalHits(true).from(0).size(1)) | ||
| .indices(getIndexName(INSTITUTION_RESOURCE, CENTRAL_TENANT_ID)); | ||
| var searchResponse = elasticClient.search(searchRequest, RequestOptions.DEFAULT); | ||
| var hitCount = Objects.requireNonNull(searchResponse.getHits().getTotalHits()).value; | ||
|
|
||
| assertThat(hitCount).isEqualTo(expected); | ||
| }); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters