feat(instance-search): create computed field for sorting and filterin…

…g Date1 (#646)

* feat(instance-search): create computed field for sorting and filtering Date1

NEWS.md

@@ -36,6 +36,7 @@
 * Implement Reindexing of Campuses ([MSEARCH-767](https://issues.folio.org/browse/MSEARCH-767))
 * Implement Reindexing of Libraries ([MSEARCH-766](https://issues.folio.org/browse/MSEARCH-766))
 * Implement Reindexing of Institutions ([MSEARCH-768](https://issues.folio.org/browse/MSEARCH-768))
+* Create computed field for sorting and filtering Date 1 ([MSEARCH-806](https://folio-org.atlassian.net/browse/MSEARCH-806))
 
 ### Bug fixes
 * Do not delete kafka topics if collection topic is enabled ([MSEARCH-725](https://folio-org.atlassian.net/browse/MSEARCH-725))

README.md

@@ -541,7 +541,7 @@ does not produce any values, so the following search options will return an empt
 | `modeOfIssuanceId`                     |   term    | `modeOfIssuanceId=="123"`                                            | Matches instances that have `123` mode of issuance                                                                                                |
 | `natureOfContentTermIds`               |   term    | `natureOfContentTermIds=="123"`                                      | Matches instances that have `123` nature of content                                                                                               |
 | `publisher`                            | full-text | `publisher all "Publisher of Ukraine"`                               | Matches instances that have `Publisher of Ukraine` publisher                                                                                      |
-| `publication.place`                    | full-text   | `publication.place all "Ukraine"`                                       | Matches instances that have `Ukraine` in publication place                                                                                           |
+| `publication.place`                    | full-text   | `publication.place all "Ukraine"`                                    | Matches instances that have `Ukraine` in publication place                                                                                        |
 | `instanceTags`                         |   term    | `instanceTags=="important"`                                          | Matches instances that have `important` tag                                                                                                       |
 | `classifications.classificationNumber` |   term    | `classifications.classificationNumber=="cl1"`                        | Matches instances that have `cl1` classification number                                                                                           |
 | `classifications.classificationTypeId` |   term    | `classifications.classificationTypeId=="123"`                        | Matches instances that have classification type id `123`                                                                                          |
@@ -559,6 +559,9 @@ does not produce any values, so the following search options will return an empt
 | `oclc`                                 |   term    | `oclc="1234*"`                                                       | Matches instances that have an OCLC identifier with the given value                                                                               |
 | `lccn`                                 |   term    | `lccn = "LCCN"`                                                      | Matches instances with the given lccn                                                                                                             |
 | `normalizedClassificationNumber`       |   term    | `normalizedClassificationNumber == "LCCN"`                           | Matches instances with the given classification number (normalizes case, whitespaces, special characters, supports leading and trailing wildcard) |
+| `dates.date1`                          |   term    | `dates.date1="199*"`                                                 | Matches instances with the given Date1 (supports leading, trailing  and internal wildcards)                                                       |
+| `dates.date2`                          |   term    | `dates.date2="199*"`                                                 | Matches instances with the given Date2 (supports leading, trailing  and internal wildcards)                                                       |
+| `normalizedDate1`                      |   term    | `normalizedDate1>=1990`                                              | Matches instances with the given Date1 (normalizes  alpha 'u' characters)                                                                         |
 
 ##### Holdings search options
 
@@ -778,12 +781,13 @@ In case where options are similar, secondary sort is used
 
 ##### Instance sort options
 
-| Option              |   Type    | Secondary sort | Description                    |
-|:--------------------|:---------:|:---------------|:-------------------------------|
-| `title`             | full text | relevancy      | Sort instances by title        |
-| `contributors`      |   term    | relevancy      | Sort instances by contributors |
-| `items.status.name` |   term    | `title`        | Sort instances by status       |
-| `item.status.name`  |   term    | `title`        | Sort instances by status       |
+| Option              |   Type    | Secondary sort | Description                        |
+|:--------------------|:---------:|:---------------|:-----------------------------------|
+| `title`             | full text | relevancy      | Sort instances by title            |
+| `contributors`      |   term    | relevancy      | Sort instances by contributors     |
+| `items.status.name` |   term    | `title`        | Sort instances by status           |
+| `item.status.name`  |   term    | `title`        | Sort instances by status           |
+| `normalizedDate1`   |   term    | relevancy      | Sort instances by normalizedDate1  |
 
 ##### Authority sort options
 

src/main/java/org/folio/search/service/setter/instance/Date1FieldProcessor.java

@@ -0,0 +1,34 @@
+package org.folio.search.service.setter.instance;
+
+import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
+import org.folio.search.domain.dto.Dates;
+import org.folio.search.domain.dto.Instance;
+import org.folio.search.service.setter.FieldProcessor;
+import org.springframework.stereotype.Component;
+
+@Component
+public class Date1FieldProcessor implements FieldProcessor<Instance, Short> {
+
+  private static final Pattern NUMERIC_REGEX = Pattern.compile("^\\d{1,4}$");
+  private static final String ZERO = "0";
+  private static final String ALPHA_U = "u";
+
+  @Override
+  public Short getFieldValue(Instance instance) {
+    Dates dates = instance.getDates();
+    if (dates != null && StringUtils.isNotEmpty(dates.getDate1())) {
+      return normalizeDate1(dates.getDate1());
+    }
+    return 0;
+  }
+
+  public Short normalizeDate1(String value) {
+    String date1 = value.replace(ALPHA_U, ZERO);
+    var matcher = NUMERIC_REGEX.matcher(date1);
+    if (matcher.find()) {
+      return Short.valueOf(matcher.group());
+    }
+    return 0;
+  }
+}

src/main/resources/elasticsearch/index-field-types.json

@@ -14,6 +14,11 @@
       "type": "long"
     }
   },
+  "short": {
+    "mapping": {
+      "type": "short"
+    }
+  },
   "keyword_uppercase": {
     "mapping": {
       "type": "keyword",

src/main/resources/model/instance.json

@@ -127,6 +127,23 @@
         }
       }
     },
+    "dates": {
+      "type": "object",
+      "properties": {
+        "dateTypeId": {
+          "index": "source",
+          "showInResponse": [ "search"]
+        },
+        "date1": {
+          "index": "source",
+          "showInResponse": [ "search" ]
+        },
+        "date2": {
+          "index": "source",
+          "showInResponse": [ "search" ]
+        }
+      }
+    },
     "instanceTypeId": {
       "searchTypes": [ "facet", "filter" ],
       "index": "keyword_lowercase"
@@ -552,6 +569,16 @@
       "index": "keyword_lowercase",
       "processor": "sortTitleProcessor"
     },
+    "normalizedDate1": {
+      "searchTypes": [ "filter", "sort" ],
+      "type": "search",
+      "index": "short",
+      "processor": "date1FieldProcessor",
+      "sort": {
+        "fieldName": "normalizedDate1",
+        "type": "single"
+      }
+    },
     "sort_contributors": {
       "searchTypes": "sort",
       "type": "search",
@@ -730,6 +757,7 @@
       "allHoldings", "plain_allHoldings",
 
       "sort_title", "sort_contributors",
+      "normalizedDate1",
       "callNumber",
       "typedCallNumber",
       "itemEffectiveShelvingOrder",

src/main/resources/swagger.api/schemas/dto/instance/dates.yaml

@@ -0,0 +1,13 @@
+type: object
+description: "Instance Dates"
+additionalProperties: false
+properties:
+  dateTypeId:
+    type: string
+    description: "Date type ID"
+  date1:
+    type: string
+    description: "Date1 value"
+  date2:
+    type: string
+    description: "Date2 value"

src/main/resources/swagger.api/schemas/entity/instance.yaml

@@ -56,6 +56,10 @@ properties:
     description: "List of subject headings"
     items:
       $ref: "../dto/instance/subject.yaml"
+  dates:
+    type: object
+    description: "Instance Dates"
+    $ref: "../dto/instance/dates.yaml"
   instanceTypeId:
     description: "UUID of the unique term for the resource type whether it's from the RDA content term list of locally defined"
     type: string

src/test/java/org/folio/search/controller/SearchByAllFieldsIT.java

@@ -42,6 +42,14 @@ static void cleanUp() {
     "Cambridge, Mass.",
     "MIT Press",
     "c2004",
+    "199u",
+    "*99u",
+    "1*9u",
+    "199*",
+    "2022",
+    "202*",
+    "2*22",
+    "*22",
 
     // holding field values
     "e3ff6133-b9a2-4d4c-a1c9-dc1867d4df19",
@@ -80,7 +88,15 @@ void canSearchByAllFieldValues_positive(String cqlQuery) throws Throwable {
     "2020-12-08T15:47:13.625+00:00",
     "2020-12-08T15:47:13.625+0000",
     "MIT Press",
-    "c2004"
+    "c2004",
+    "199u",
+    "*99u",
+    "1*9u",
+    "199*",
+    "2022",
+    "202*",
+    "2*22",
+    "*22"
   })
   @ParameterizedTest(name = "[{index}] cql.allInstances='{query}', query=''{0}''")
   void canSearchByInstanceFieldValues_positive(String cqlQuery) throws Throwable {

src/test/java/org/folio/search/controller/SearchInstanceFilterIT.java

@@ -23,6 +23,7 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import org.folio.search.domain.dto.Classification;
+import org.folio.search.domain.dto.Dates;
 import org.folio.search.domain.dto.Facet;
 import org.folio.search.domain.dto.FacetResult;
 import org.folio.search.domain.dto.Holding;
@@ -91,6 +92,9 @@ class SearchInstanceFilterIT extends BaseIntegrationTest {
     "5af5cb9d-063f-48ea-8148-7da3ecaafd7d",
     "7e5684a9-c8c1-4c1e-85b9-d047f53eeb6d");
 
+  private static final String[] DATES = array(
+    "2021", "1998", "2020", "1978", "2023", "2022");
+
   @BeforeAll
   static void prepare() {
     setUpTenant(instances());
@@ -202,6 +206,13 @@ private static Stream<Arguments> filteredSearchQueriesProvider() {
       arguments(format("(item.effectiveLocationId==%s) sortby title", LOCATIONS[0]),
         List.of(IDS[0], IDS[2], IDS[3], IDS[4])),
 
+      arguments(format("(normalizedDate1<%s) sortby normalizedDate1", DATES[2]), List.of(IDS[3], IDS[1])),
+      arguments(format("(normalizedDate1>=%s and normalizedDate1<%s) sortby title", DATES[1], DATES[5]),
+        List.of(IDS[0], IDS[1], IDS[2])),
+      arguments(format("(normalizedDate1>=%s and normalizedDate1<%s) sortby normalizedDate1", DATES[1], DATES[5]),
+        List.of(IDS[1], IDS[2], IDS[0])),
+      arguments(format("(normalizedDate1>=%s) sortby title", DATES[0]), List.of(IDS[0], IDS[4])),
+
       arguments("(item.status.name==Available) sortby title", List.of(IDS[0], IDS[1], IDS[4])),
       arguments("(item.status.name==Missing) sortby title", List.of(IDS[2], IDS[3])),
       arguments("(item.status.name==\"Checked out\") sortby title", List.of(IDS[2], IDS[4])),
@@ -484,6 +495,7 @@ private static Instance[] instances() {
       .tags(tags("text", "science"))
       .statisticalCodeIds(singletonList("b5968c9e-cddc-4576-99e3-8e60aed8b0dd"))
       .metadata(metadata("2021-03-01T00:00:00.000+00:00", "2021-03-05T12:30:00.000+00:00"))
+      .dates(new Dates().date1(DATES[0]))
       .items(List.of(
         new Item().id(randomId())
           .effectiveLocationId(LOCATIONS[0]).status(itemStatus(AVAILABLE))
@@ -511,6 +523,7 @@ private static Instance[] instances() {
       .instanceFormatIds(List.of(FORMATS[1]))
       .tags(tags("future"))
       .metadata(metadata("2021-03-10T01:00:00.000+00:00", "2021-03-12T15:40:00.000+00:00"))
+      .dates(new Dates().date1(DATES[1]))
       .items(List.of(
         new Item().id(randomId())
           .effectiveLocationId(LOCATIONS[1]).status(itemStatus(AVAILABLE))
@@ -537,6 +550,7 @@ private static Instance[] instances() {
       .instanceFormatIds(List.of(FORMATS[2]))
       .tags(tags("future", "science"))
       .metadata(metadata("2021-03-08T15:00:00.000+00:00", "2021-03-15T22:30:00.000+00:00"))
+      .dates(new Dates().date1(DATES[2]))
       .items(List.of(
         new Item().id(randomId()).effectiveLocationId(LOCATIONS[0]).status(itemStatus(MISSING))
           .metadata(metadata("2021-03-08T15:00:00.000+00:00", "2021-03-15T22:30:00.000+00:00"))
@@ -555,6 +569,7 @@ private static Instance[] instances() {
       .instanceFormatIds(List.of(FORMATS))
       .tags(tags("casual", "cooking"))
       .metadata(metadata("2021-03-15T12:00:00.000+00:00", "2021-03-15T12:00:00.000+00:00"))
+      .dates(new Dates().date1(DATES[3]))
       .items(List.of(new Item().id(randomId())
         .effectiveLocationId(LOCATIONS[0]).status(itemStatus(MISSING))
         .metadata(metadata("2021-03-15T12:00:00.000+00:00", "2021-03-15T12:00:00.000+00:00"))
@@ -577,6 +592,7 @@ private static Instance[] instances() {
       .statusId(STATUSES[1])
       .instanceFormatIds(List.of(FORMATS[1]))
       .tags(tags("cooking"))
+      .dates(new Dates().date1(DATES[4]).date2(DATES[4]))
       .items(List.of(
         new Item().id(randomId()).effectiveLocationId(LOCATIONS[0]).status(itemStatus(CHECKED_OUT)).tags(tags("itag3")),
         new Item().id(randomId()).effectiveLocationId(LOCATIONS[1]).status(itemStatus(AVAILABLE))

src/test/java/org/folio/search/controller/SearchInstanceIT.java

@@ -465,6 +465,16 @@ private static Stream<Arguments> testCaseInsensitiveDataProvider() {
       arguments("contributors.name = {value}", "ANTON*"),
       arguments("contributors.name = {value}", "*RMELEN, FRANK"),
 
+      arguments("dates.date1 == {value}", "*9u"),
+      arguments("dates.date1 == {value}", "*99*"),
+      arguments("dates.date1 == {value}", "19*"),
+      arguments("dates.date1 > {value}", "19*"),
+
+      arguments("dates.date2 == {value}", "*22"),
+      arguments("dates.date2 == {value}", "*02*"),
+      arguments("dates.date2 == {value}", "20*"),
+      arguments("dates.date2 > {value}", " 2021"),
+
       arguments("contributors.authorityId == {value}", "55294032-FCF6-45CC-B6DA-4420A61EF72C"),
       arguments("authorityId == {value}", "55294032-FCF6-45CC-B6DA-4420A61EF72C"),
 

src/test/java/org/folio/search/controller/SortInstanceIT.java

@@ -10,6 +10,7 @@
 
 import java.util.ArrayList;
 import org.folio.search.domain.dto.Contributor;
+import org.folio.search.domain.dto.Dates;
 import org.folio.search.domain.dto.Instance;
 import org.folio.search.support.base.BaseIntegrationTest;
 import org.folio.spring.testing.type.IntegrationTest;
@@ -40,6 +41,26 @@ void canSortInstancesByContributors_asc() throws Exception {
       .andExpect(jsonPath("instances[4].contributors[0].name", is("yyy zzz")));
   }
 
+  @Test
+  void canSortInstancesByDate1_asc() throws Exception {
+    doSearchByInstances(allRecordsSortedBy("normalizedDate1", ASCENDING)).andExpect(jsonPath("totalRecords", is(5)))
+      .andExpect(jsonPath("instances[0].dates.date1", is("19u5")))
+      .andExpect(jsonPath("instances[1].dates.date1", is("199u")))
+      .andExpect(jsonPath("instances[2].dates.date1", is("1999")))
+      .andExpect(jsonPath("instances[3].dates.date1", is("2001")))
+      .andExpect(jsonPath("instances[4].dates.date1", is("2021")));
+  }
+
+  @Test
+  void canSortInstancesByDate1_desc() throws Exception {
+    doSearchByInstances(allRecordsSortedBy("normalizedDate1", DESCENDING)).andExpect(jsonPath("totalRecords", is(5)))
+      .andExpect(jsonPath("instances[0].dates.date1", is("2021")))
+      .andExpect(jsonPath("instances[1].dates.date1", is("2001")))
+      .andExpect(jsonPath("instances[2].dates.date1", is("1999")))
+      .andExpect(jsonPath("instances[3].dates.date1", is("199u")))
+      .andExpect(jsonPath("instances[4].dates.date1", is("19u5")));
+  }
+
   @Test
   void canSortInstancesByContributors_desc() throws Exception {
     doSearchByInstances(allRecordsSortedBy("contributors", DESCENDING)).andExpect(jsonPath("totalRecords", is(5)))
@@ -90,23 +111,35 @@ private static Instance[] instances() {
 
     instances[0].title("Animal farm")
       .indexTitle("B1 Animal farm")
-      .addContributorsItem(new Contributor().name("yyy zzz"));
+      .addContributorsItem(new Contributor().name("yyy zzz"))
+      .setDates(getDates("1999", "2000"));
 
     instances[1].title("Zero Minus Ten")
       .indexTitle(null)
       .addContributorsItem(new Contributor().name("aaa bbb").primary(false))
-      .addContributorsItem(new Contributor().name("bbb ccc").primary(true));
+      .addContributorsItem(new Contributor().name("bbb ccc").primary(true))
+      .setDates(getDates("199u", "2000"));
 
     instances[2].title("Calling Me Home")
       .indexTitle("A1 Calling Me Home")
-      .addContributorsItem(new Contributor().name("bcc ccc"));
+      .addContributorsItem(new Contributor().name("bcc ccc"))
+      .setDates(getDates("2021", "2022"));
 
     instances[3].title("Walk in My Soul")
       .indexTitle(null)
-      .addContributorsItem(new Contributor().name("1111 2222").primary(true));
+      .addContributorsItem(new Contributor().name("1111 2222").primary(true))
+      .setDates(getDates("2001", "2002"));
 
-    instances[4].title("Star Wars").indexTitle(null).addContributorsItem(new Contributor().name("Śląsk").primary(true));
+    instances[4].title("Star Wars").indexTitle(null).addContributorsItem(new Contributor().name("Śląsk").primary(true))
+      .setDates(getDates("19u5", "1998"));
 
     return instances;
   }
+
+  private static Dates getDates(String date1, String date2) {
+    Dates dates = new Dates();
+    dates.setDate1(date1);
+    dates.setDate2(date2);
+    return dates;
+  }
 }