diff --git a/NEWS.md b/NEWS.md index a653bd139..238a095f6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -36,6 +36,7 @@ * Implement Reindexing of Campuses ([MSEARCH-767](https://issues.folio.org/browse/MSEARCH-767)) * Implement Reindexing of Libraries ([MSEARCH-766](https://issues.folio.org/browse/MSEARCH-766)) * Implement Reindexing of Institutions ([MSEARCH-768](https://issues.folio.org/browse/MSEARCH-768)) +* Create computed field for sorting and filtering Date 1 ([MSEARCH-806](https://folio-org.atlassian.net/browse/MSEARCH-806)) ### Bug fixes * Do not delete kafka topics if collection topic is enabled ([MSEARCH-725](https://folio-org.atlassian.net/browse/MSEARCH-725)) diff --git a/README.md b/README.md index b2bac11fb..d83b76896 100644 --- a/README.md +++ b/README.md @@ -541,7 +541,7 @@ does not produce any values, so the following search options will return an empt | `modeOfIssuanceId` | term | `modeOfIssuanceId=="123"` | Matches instances that have `123` mode of issuance | | `natureOfContentTermIds` | term | `natureOfContentTermIds=="123"` | Matches instances that have `123` nature of content | | `publisher` | full-text | `publisher all "Publisher of Ukraine"` | Matches instances that have `Publisher of Ukraine` publisher | -| `publication.place` | full-text | `publication.place all "Ukraine"` | Matches instances that have `Ukraine` in publication place | +| `publication.place` | full-text | `publication.place all "Ukraine"` | Matches instances that have `Ukraine` in publication place | | `instanceTags` | term | `instanceTags=="important"` | Matches instances that have `important` tag | | `classifications.classificationNumber` | term | `classifications.classificationNumber=="cl1"` | Matches instances that have `cl1` classification number | | `classifications.classificationTypeId` | term | `classifications.classificationTypeId=="123"` | Matches instances that have classification type id `123` | @@ -559,6 +559,9 @@ does not produce any values, so the following search options will return an empt | `oclc` | term | `oclc="1234*"` | Matches instances that have an OCLC identifier with the given value | | `lccn` | term | `lccn = "LCCN"` | Matches instances with the given lccn | | `normalizedClassificationNumber` | term | `normalizedClassificationNumber == "LCCN"` | Matches instances with the given classification number (normalizes case, whitespaces, special characters, supports leading and trailing wildcard) | +| `dates.date1` | term | `dates.date1="199*"` | Matches instances with the given Date1 (supports leading, trailing and internal wildcards) | +| `dates.date2` | term | `dates.date2="199*"` | Matches instances with the given Date2 (supports leading, trailing and internal wildcards) | +| `normalizedDate1` | term | `normalizedDate1>=1990` | Matches instances with the given Date1 (normalizes alpha 'u' characters) | ##### Holdings search options @@ -778,12 +781,13 @@ In case where options are similar, secondary sort is used ##### Instance sort options -| Option | Type | Secondary sort | Description | -|:--------------------|:---------:|:---------------|:-------------------------------| -| `title` | full text | relevancy | Sort instances by title | -| `contributors` | term | relevancy | Sort instances by contributors | -| `items.status.name` | term | `title` | Sort instances by status | -| `item.status.name` | term | `title` | Sort instances by status | +| Option | Type | Secondary sort | Description | +|:--------------------|:---------:|:---------------|:-----------------------------------| +| `title` | full text | relevancy | Sort instances by title | +| `contributors` | term | relevancy | Sort instances by contributors | +| `items.status.name` | term | `title` | Sort instances by status | +| `item.status.name` | term | `title` | Sort instances by status | +| `normalizedDate1` | term | relevancy | Sort instances by normalizedDate1 | ##### Authority sort options diff --git a/src/main/java/org/folio/search/service/setter/instance/Date1FieldProcessor.java b/src/main/java/org/folio/search/service/setter/instance/Date1FieldProcessor.java new file mode 100644 index 000000000..772210af3 --- /dev/null +++ b/src/main/java/org/folio/search/service/setter/instance/Date1FieldProcessor.java @@ -0,0 +1,34 @@ +package org.folio.search.service.setter.instance; + +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.folio.search.domain.dto.Dates; +import org.folio.search.domain.dto.Instance; +import org.folio.search.service.setter.FieldProcessor; +import org.springframework.stereotype.Component; + +@Component +public class Date1FieldProcessor implements FieldProcessor { + + private static final Pattern NUMERIC_REGEX = Pattern.compile("^\\d{1,4}$"); + private static final String ZERO = "0"; + private static final String ALPHA_U = "u"; + + @Override + public Short getFieldValue(Instance instance) { + Dates dates = instance.getDates(); + if (dates != null && StringUtils.isNotEmpty(dates.getDate1())) { + return normalizeDate1(dates.getDate1()); + } + return 0; + } + + public Short normalizeDate1(String value) { + String date1 = value.replace(ALPHA_U, ZERO); + var matcher = NUMERIC_REGEX.matcher(date1); + if (matcher.find()) { + return Short.valueOf(matcher.group()); + } + return 0; + } +} diff --git a/src/main/resources/elasticsearch/index-field-types.json b/src/main/resources/elasticsearch/index-field-types.json index 012c7c6b4..eb0c0c377 100644 --- a/src/main/resources/elasticsearch/index-field-types.json +++ b/src/main/resources/elasticsearch/index-field-types.json @@ -14,6 +14,11 @@ "type": "long" } }, + "short": { + "mapping": { + "type": "short" + } + }, "keyword_uppercase": { "mapping": { "type": "keyword", diff --git a/src/main/resources/model/instance.json b/src/main/resources/model/instance.json index e0fe86cf2..e8ff0d557 100644 --- a/src/main/resources/model/instance.json +++ b/src/main/resources/model/instance.json @@ -127,6 +127,23 @@ } } }, + "dates": { + "type": "object", + "properties": { + "dateTypeId": { + "index": "source", + "showInResponse": [ "search"] + }, + "date1": { + "index": "source", + "showInResponse": [ "search" ] + }, + "date2": { + "index": "source", + "showInResponse": [ "search" ] + } + } + }, "instanceTypeId": { "searchTypes": [ "facet", "filter" ], "index": "keyword_lowercase" @@ -552,6 +569,16 @@ "index": "keyword_lowercase", "processor": "sortTitleProcessor" }, + "normalizedDate1": { + "searchTypes": [ "filter", "sort" ], + "type": "search", + "index": "short", + "processor": "date1FieldProcessor", + "sort": { + "fieldName": "normalizedDate1", + "type": "single" + } + }, "sort_contributors": { "searchTypes": "sort", "type": "search", @@ -730,6 +757,7 @@ "allHoldings", "plain_allHoldings", "sort_title", "sort_contributors", + "normalizedDate1", "callNumber", "typedCallNumber", "itemEffectiveShelvingOrder", diff --git a/src/main/resources/swagger.api/schemas/dto/instance/dates.yaml b/src/main/resources/swagger.api/schemas/dto/instance/dates.yaml new file mode 100644 index 000000000..15a603a66 --- /dev/null +++ b/src/main/resources/swagger.api/schemas/dto/instance/dates.yaml @@ -0,0 +1,13 @@ +type: object +description: "Instance Dates" +additionalProperties: false +properties: + dateTypeId: + type: string + description: "Date type ID" + date1: + type: string + description: "Date1 value" + date2: + type: string + description: "Date2 value" diff --git a/src/main/resources/swagger.api/schemas/entity/instance.yaml b/src/main/resources/swagger.api/schemas/entity/instance.yaml index 9b8827fb1..bf9f55828 100644 --- a/src/main/resources/swagger.api/schemas/entity/instance.yaml +++ b/src/main/resources/swagger.api/schemas/entity/instance.yaml @@ -56,6 +56,10 @@ properties: description: "List of subject headings" items: $ref: "../dto/instance/subject.yaml" + dates: + type: object + description: "Instance Dates" + $ref: "../dto/instance/dates.yaml" instanceTypeId: description: "UUID of the unique term for the resource type whether it's from the RDA content term list of locally defined" type: string diff --git a/src/test/java/org/folio/search/controller/SearchByAllFieldsIT.java b/src/test/java/org/folio/search/controller/SearchByAllFieldsIT.java index feef40a3e..ef05647bb 100644 --- a/src/test/java/org/folio/search/controller/SearchByAllFieldsIT.java +++ b/src/test/java/org/folio/search/controller/SearchByAllFieldsIT.java @@ -42,6 +42,14 @@ static void cleanUp() { "Cambridge, Mass.", "MIT Press", "c2004", + "199u", + "*99u", + "1*9u", + "199*", + "2022", + "202*", + "2*22", + "*22", // holding field values "e3ff6133-b9a2-4d4c-a1c9-dc1867d4df19", @@ -80,7 +88,15 @@ void canSearchByAllFieldValues_positive(String cqlQuery) throws Throwable { "2020-12-08T15:47:13.625+00:00", "2020-12-08T15:47:13.625+0000", "MIT Press", - "c2004" + "c2004", + "199u", + "*99u", + "1*9u", + "199*", + "2022", + "202*", + "2*22", + "*22" }) @ParameterizedTest(name = "[{index}] cql.allInstances='{query}', query=''{0}''") void canSearchByInstanceFieldValues_positive(String cqlQuery) throws Throwable { diff --git a/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java b/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java index 24b3b75ff..535eb92b6 100644 --- a/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java +++ b/src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java @@ -23,6 +23,7 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import org.folio.search.domain.dto.Classification; +import org.folio.search.domain.dto.Dates; import org.folio.search.domain.dto.Facet; import org.folio.search.domain.dto.FacetResult; import org.folio.search.domain.dto.Holding; @@ -91,6 +92,9 @@ class SearchInstanceFilterIT extends BaseIntegrationTest { "5af5cb9d-063f-48ea-8148-7da3ecaafd7d", "7e5684a9-c8c1-4c1e-85b9-d047f53eeb6d"); + private static final String[] DATES = array( + "2021", "1998", "2020", "1978", "2023", "2022"); + @BeforeAll static void prepare() { setUpTenant(instances()); @@ -202,6 +206,13 @@ private static Stream filteredSearchQueriesProvider() { arguments(format("(item.effectiveLocationId==%s) sortby title", LOCATIONS[0]), List.of(IDS[0], IDS[2], IDS[3], IDS[4])), + arguments(format("(normalizedDate1<%s) sortby normalizedDate1", DATES[2]), List.of(IDS[3], IDS[1])), + arguments(format("(normalizedDate1>=%s and normalizedDate1<%s) sortby title", DATES[1], DATES[5]), + List.of(IDS[0], IDS[1], IDS[2])), + arguments(format("(normalizedDate1>=%s and normalizedDate1<%s) sortby normalizedDate1", DATES[1], DATES[5]), + List.of(IDS[1], IDS[2], IDS[0])), + arguments(format("(normalizedDate1>=%s) sortby title", DATES[0]), List.of(IDS[0], IDS[4])), + arguments("(item.status.name==Available) sortby title", List.of(IDS[0], IDS[1], IDS[4])), arguments("(item.status.name==Missing) sortby title", List.of(IDS[2], IDS[3])), arguments("(item.status.name==\"Checked out\") sortby title", List.of(IDS[2], IDS[4])), @@ -484,6 +495,7 @@ private static Instance[] instances() { .tags(tags("text", "science")) .statisticalCodeIds(singletonList("b5968c9e-cddc-4576-99e3-8e60aed8b0dd")) .metadata(metadata("2021-03-01T00:00:00.000+00:00", "2021-03-05T12:30:00.000+00:00")) + .dates(new Dates().date1(DATES[0])) .items(List.of( new Item().id(randomId()) .effectiveLocationId(LOCATIONS[0]).status(itemStatus(AVAILABLE)) @@ -511,6 +523,7 @@ private static Instance[] instances() { .instanceFormatIds(List.of(FORMATS[1])) .tags(tags("future")) .metadata(metadata("2021-03-10T01:00:00.000+00:00", "2021-03-12T15:40:00.000+00:00")) + .dates(new Dates().date1(DATES[1])) .items(List.of( new Item().id(randomId()) .effectiveLocationId(LOCATIONS[1]).status(itemStatus(AVAILABLE)) @@ -537,6 +550,7 @@ private static Instance[] instances() { .instanceFormatIds(List.of(FORMATS[2])) .tags(tags("future", "science")) .metadata(metadata("2021-03-08T15:00:00.000+00:00", "2021-03-15T22:30:00.000+00:00")) + .dates(new Dates().date1(DATES[2])) .items(List.of( new Item().id(randomId()).effectiveLocationId(LOCATIONS[0]).status(itemStatus(MISSING)) .metadata(metadata("2021-03-08T15:00:00.000+00:00", "2021-03-15T22:30:00.000+00:00")) @@ -555,6 +569,7 @@ private static Instance[] instances() { .instanceFormatIds(List.of(FORMATS)) .tags(tags("casual", "cooking")) .metadata(metadata("2021-03-15T12:00:00.000+00:00", "2021-03-15T12:00:00.000+00:00")) + .dates(new Dates().date1(DATES[3])) .items(List.of(new Item().id(randomId()) .effectiveLocationId(LOCATIONS[0]).status(itemStatus(MISSING)) .metadata(metadata("2021-03-15T12:00:00.000+00:00", "2021-03-15T12:00:00.000+00:00")) @@ -577,6 +592,7 @@ private static Instance[] instances() { .statusId(STATUSES[1]) .instanceFormatIds(List.of(FORMATS[1])) .tags(tags("cooking")) + .dates(new Dates().date1(DATES[4]).date2(DATES[4])) .items(List.of( new Item().id(randomId()).effectiveLocationId(LOCATIONS[0]).status(itemStatus(CHECKED_OUT)).tags(tags("itag3")), new Item().id(randomId()).effectiveLocationId(LOCATIONS[1]).status(itemStatus(AVAILABLE)) diff --git a/src/test/java/org/folio/search/controller/SearchInstanceIT.java b/src/test/java/org/folio/search/controller/SearchInstanceIT.java index 98e182157..34d918f4e 100644 --- a/src/test/java/org/folio/search/controller/SearchInstanceIT.java +++ b/src/test/java/org/folio/search/controller/SearchInstanceIT.java @@ -465,6 +465,16 @@ private static Stream testCaseInsensitiveDataProvider() { arguments("contributors.name = {value}", "ANTON*"), arguments("contributors.name = {value}", "*RMELEN, FRANK"), + arguments("dates.date1 == {value}", "*9u"), + arguments("dates.date1 == {value}", "*99*"), + arguments("dates.date1 == {value}", "19*"), + arguments("dates.date1 > {value}", "19*"), + + arguments("dates.date2 == {value}", "*22"), + arguments("dates.date2 == {value}", "*02*"), + arguments("dates.date2 == {value}", "20*"), + arguments("dates.date2 > {value}", " 2021"), + arguments("contributors.authorityId == {value}", "55294032-FCF6-45CC-B6DA-4420A61EF72C"), arguments("authorityId == {value}", "55294032-FCF6-45CC-B6DA-4420A61EF72C"), diff --git a/src/test/java/org/folio/search/controller/SortInstanceIT.java b/src/test/java/org/folio/search/controller/SortInstanceIT.java index e42e7b6b8..e6fd71739 100644 --- a/src/test/java/org/folio/search/controller/SortInstanceIT.java +++ b/src/test/java/org/folio/search/controller/SortInstanceIT.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import org.folio.search.domain.dto.Contributor; +import org.folio.search.domain.dto.Dates; import org.folio.search.domain.dto.Instance; import org.folio.search.support.base.BaseIntegrationTest; import org.folio.spring.testing.type.IntegrationTest; @@ -40,6 +41,26 @@ void canSortInstancesByContributors_asc() throws Exception { .andExpect(jsonPath("instances[4].contributors[0].name", is("yyy zzz"))); } + @Test + void canSortInstancesByDate1_asc() throws Exception { + doSearchByInstances(allRecordsSortedBy("normalizedDate1", ASCENDING)).andExpect(jsonPath("totalRecords", is(5))) + .andExpect(jsonPath("instances[0].dates.date1", is("19u5"))) + .andExpect(jsonPath("instances[1].dates.date1", is("199u"))) + .andExpect(jsonPath("instances[2].dates.date1", is("1999"))) + .andExpect(jsonPath("instances[3].dates.date1", is("2001"))) + .andExpect(jsonPath("instances[4].dates.date1", is("2021"))); + } + + @Test + void canSortInstancesByDate1_desc() throws Exception { + doSearchByInstances(allRecordsSortedBy("normalizedDate1", DESCENDING)).andExpect(jsonPath("totalRecords", is(5))) + .andExpect(jsonPath("instances[0].dates.date1", is("2021"))) + .andExpect(jsonPath("instances[1].dates.date1", is("2001"))) + .andExpect(jsonPath("instances[2].dates.date1", is("1999"))) + .andExpect(jsonPath("instances[3].dates.date1", is("199u"))) + .andExpect(jsonPath("instances[4].dates.date1", is("19u5"))); + } + @Test void canSortInstancesByContributors_desc() throws Exception { doSearchByInstances(allRecordsSortedBy("contributors", DESCENDING)).andExpect(jsonPath("totalRecords", is(5))) @@ -90,23 +111,35 @@ private static Instance[] instances() { instances[0].title("Animal farm") .indexTitle("B1 Animal farm") - .addContributorsItem(new Contributor().name("yyy zzz")); + .addContributorsItem(new Contributor().name("yyy zzz")) + .setDates(getDates("1999", "2000")); instances[1].title("Zero Minus Ten") .indexTitle(null) .addContributorsItem(new Contributor().name("aaa bbb").primary(false)) - .addContributorsItem(new Contributor().name("bbb ccc").primary(true)); + .addContributorsItem(new Contributor().name("bbb ccc").primary(true)) + .setDates(getDates("199u", "2000")); instances[2].title("Calling Me Home") .indexTitle("A1 Calling Me Home") - .addContributorsItem(new Contributor().name("bcc ccc")); + .addContributorsItem(new Contributor().name("bcc ccc")) + .setDates(getDates("2021", "2022")); instances[3].title("Walk in My Soul") .indexTitle(null) - .addContributorsItem(new Contributor().name("1111 2222").primary(true)); + .addContributorsItem(new Contributor().name("1111 2222").primary(true)) + .setDates(getDates("2001", "2002")); - instances[4].title("Star Wars").indexTitle(null).addContributorsItem(new Contributor().name("Śląsk").primary(true)); + instances[4].title("Star Wars").indexTitle(null).addContributorsItem(new Contributor().name("Śląsk").primary(true)) + .setDates(getDates("19u5", "1998")); return instances; } + + private static Dates getDates(String date1, String date2) { + Dates dates = new Dates(); + dates.setDate1(date1); + dates.setDate2(date2); + return dates; + } } diff --git a/src/test/java/org/folio/search/service/setter/instance/Date1FieldProcessorTest.java b/src/test/java/org/folio/search/service/setter/instance/Date1FieldProcessorTest.java new file mode 100644 index 000000000..7cc6bdbc0 --- /dev/null +++ b/src/test/java/org/folio/search/service/setter/instance/Date1FieldProcessorTest.java @@ -0,0 +1,48 @@ +package org.folio.search.service.setter.instance; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.util.stream.Stream; +import org.folio.search.domain.dto.Dates; +import org.folio.search.domain.dto.Instance; +import org.folio.spring.testing.type.UnitTest; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@UnitTest +class Date1FieldProcessorTest { + + private final Date1FieldProcessor date1FieldProcessor = new Date1FieldProcessor(); + + @MethodSource("date1DataProvider") + @DisplayName("getFieldValue_parameterized") + @ParameterizedTest(name = "[{index}] instance with {0}, expected={1}") + void getFieldValue_parameterized(Instance eventBody, Short expected) { + var actual = date1FieldProcessor.getFieldValue(eventBody); + assertThat(actual).isEqualTo(expected); + } + + private static Stream date1DataProvider() { + return Stream.of( + arguments(instance("1999"), Short.valueOf("1999")), + arguments(instance("0999"), Short.valueOf("999")), + arguments(instance("uuu9"), Short.valueOf("9")), + arguments(instance("199u"), Short.valueOf("1990")), + arguments(instance("20u2"), Short.valueOf("2002")), + arguments(instance("20uu"), Short.valueOf("2000")), + arguments(instance("199"), Short.valueOf("199")), + arguments(instance("1u9"), Short.valueOf("109")), + arguments(instance("1"), Short.valueOf("1")), + arguments(instance("19999"), Short.valueOf("0")), + arguments(instance("19k5"), Short.valueOf("0")), + arguments(new Instance(), Short.valueOf("0")) + ); + } + + private static Instance instance(String date1) { + return new Instance().dates(new Dates().date1(date1)); + } +} diff --git a/src/test/resources/samples/instance-response-sample/instance-basic-response.json b/src/test/resources/samples/instance-response-sample/instance-basic-response.json index b95856f13..18d19ce53 100644 --- a/src/test/resources/samples/instance-response-sample/instance-basic-response.json +++ b/src/test/resources/samples/instance-response-sample/instance-basic-response.json @@ -26,6 +26,11 @@ "dateOfPublication": "c2004" } ], + "dates": { + "dateTypeId": "0750f52b-3bfc-458d-9307-e9afc8bcdffa", + "date1": "199u", + "date2": "2022" + }, "staffSuppress": false, "discoverySuppress": false, "isBoundWith": true, @@ -66,4 +71,4 @@ "holdings": [] } ] -} \ No newline at end of file +} diff --git a/src/test/resources/samples/instance-response-sample/instance-full-response.json b/src/test/resources/samples/instance-response-sample/instance-full-response.json index bb29ff358..a2064d091 100644 --- a/src/test/resources/samples/instance-response-sample/instance-full-response.json +++ b/src/test/resources/samples/instance-response-sample/instance-full-response.json @@ -104,6 +104,11 @@ "authorityId": "55294032-fcf6-45cc-b6da-4420a61ef72d" } ], + "dates": { + "dateTypeId": "0750f52b-3bfc-458d-9307-e9afc8bcdffa", + "date1": "199u", + "date2": "2022" + }, "instanceTypeId": "6312d172-f0cf-40f6-b27d-9fa8feaf332f", "instanceFormatIds": [ "7f9c4ac0-fa3d-43b7-b978-3bf0be38c4da" diff --git a/src/test/resources/samples/semantic-web-primer/instance.json b/src/test/resources/samples/semantic-web-primer/instance.json index d9f557024..4d2bb0977 100644 --- a/src/test/resources/samples/semantic-web-primer/instance.json +++ b/src/test/resources/samples/semantic-web-primer/instance.json @@ -127,6 +127,11 @@ "materialsSpecification": "electronic access material" } ], + "dates": { + "dateTypeId": "0750f52b-3bfc-458d-9307-e9afc8bcdffa", + "date1": "199u", + "date2": "2022" + }, "instanceTypeId": "6312d172-f0cf-40f6-b27d-9fa8feaf332f", "instanceFormatIds": [ "7f9c4ac0-fa3d-43b7-b978-3bf0be38c4da"