Merge b1.0 into master

.gitignore

@@ -3,6 +3,7 @@ target/
 !.mvn/wrapper/maven-wrapper.jar
 !**/src/main/**/target/
 !**/src/test/**/target/
+.env
 
 ### STS ###
 .apt_generated

NEWS.md

@@ -1,5 +1,14 @@
-# 1.0.0
-- Initial release
+# 1.0.x
+
+## 1.0.3
+- [MODFQMMGR-58](https://issues.folio.org/browse/MODFQMMGR-58) Refactor drv_item_callnumber_location view
+- [MODFQMMGR-76](https://issues.folio.org/browse/MODFQMMGR-76) Periodically refresh materialized views
+
+## 1.0.2
+- Remove the instance_title_searchable field from the Items entity type
+- Purge old query results based on query start date/time instead of the end date/time
+- Fix bug in user preferred contact type
+- Update the provided `_tenant` interface in the module descriptor to 2.0
 
 ## 1.0.1
 - [MODFQMMGR-57](https://issues.folio.org/browse/MODFQMMGR-57) Use a different version of f_unaccent(), to allow us to make use of an index
@@ -8,3 +17,6 @@
 - [MODFQMMGR-67](https://issues.folio.org/browse/MODFQMMGR-67) Enable batched inserts
 - [MODFQMMGR-71](https://issues.folio.org/browse/MODFQMMGR-71) Update item and user entity types
 - [MODFQMMGR-31](https://issues.folio.org/browse/MODFQMMGR-31) Fix Users dropdown
+
+## 1.0.0
+- Initial release

descriptors/ModuleDescriptor-template.json

@@ -1,18 +1,21 @@
 {
   "id": "@artifactId@-@version@",
-  "name": "The module descriptor for mod-fqm-manager.",
+  "name": "FQM Manager Module",
   "provides": [
     {
       "id": "_tenant",
-      "version": "1.2",
+      "version": "2.0",
       "interfaceType": "system",
       "handlers": [
         {
           "methods": ["POST"],
-          "pathPattern": "/_/tenant"
-        }, {
-          "methods": ["DELETE"],
-          "pathPattern": "/_/tenant"
+          "pathPattern": "/_/tenant",
+          "permissionsRequired": []
+        },
+        {
+          "methods": ["GET", "DELETE"],
+          "pathPattern": "/_/tenant/{id}",
+          "permissionsRequired": []
         }
       ]
     },
@@ -34,6 +37,11 @@
           "methods": ["GET"],
           "pathPattern": "/entity-types/{entity-type-id}/columns/{column-name}/values",
           "permissionsRequired": ["fqm.entityTypes.item.columnValues.get"]
+        },
+        {
+          "methods": ["POST"],
+          "pathPattern": "/entity-types/materialized-views/refresh",
+          "permissionsRequired": ["fqm.materializedViews.post"]
         }
       ]
     },
@@ -84,10 +92,22 @@
       "interfaceType": "system",
       "handlers": [
         {
-          "methods": [ "POST" ],
+          "methods": ["POST"],
           "pathPattern": "/query/purge",
           "unit": "hour",
-          "delay": "1"
+          "delay": "1",
+          "modulePermissions": [
+            "fqm.query.purge"
+          ]
+        },
+        {
+          "methods": ["POST"],
+          "pathPattern": "/entity-types/materialized-views/refresh",
+          "unit": "hour",
+          "delay": "24",
+          "modulePermissions": [
+            "fqm.materializedViews.post"
+          ]
         }
       ]
     }
@@ -141,6 +161,12 @@
       "description": "Run a query synchronously and get results",
       "visible":  true
     },
+    {
+      "permissionName": "fqm.materializedViews.post",
+      "displayName": "FQM - Refresh materialized views",
+      "description": "Refresh FQM materialized views",
+      "visible":  true
+    },
     {
       "permissionName": "fqm.query.all",
       "displayName": "FQM - All permissions",

pom.xml

@@ -10,7 +10,7 @@
   <groupId>org.folio</groupId>
   <artifactId>mod-fqm-manager</artifactId>
   <name>mod-fqm-manager</name>
-  <version>1.0.2-SNAPSHOT</version>
+  <version>1.0.4-SNAPSHOT</version>
   <description>FOLIO Query Machine manager</description>
   <packaging>jar</packaging>
 
@@ -373,10 +373,10 @@
   </distributionManagement>
 
   <scm>
-    <url>https://github.com/EBSCOIS/${project.artifactId}</url>
-    <connection>scm:git:git://github.com/EBSCOIS/${project.artifactId}.git</connection>
-    <developerConnection>scm:git:[email protected]:EBSCOIS/${project.artifactId}.git</developerConnection>
-    <tag>v1.0.0</tag>
+    <url>https://github.com/folio-org/${artifactId}</url>
+    <connection>scm:git:git://github.com/folio-org/${artifactId}.git</connection>
+    <developerConnection>scm:git:[email protected]:folio-org/${artifactId}.git</developerConnection>
+    <tag>HEAD</tag>
   </scm>
 
 </project>

src/main/java/org/folio/fqm/repository/MaterializedViewRefreshRepository.java

@@ -0,0 +1,30 @@
+package org.folio.fqm.repository;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.jooq.DSLContext;
+import org.springframework.stereotype.Repository;
+
+import java.util.List;
+
+@Repository
+@RequiredArgsConstructor
+@Log4j2
+public class MaterializedViewRefreshRepository {
+  private static final String REFRESH_MATERIALIZED_VIEW_SQL = "REFRESH MATERIALIZED VIEW CONCURRENTLY ";
+
+  private static final List<String> materializedViewNames = List.of(
+    "drv_circulation_loan_status",
+    "drv_inventory_item_status"
+  );
+
+  private final DSLContext jooqContext;
+
+  public void refreshMaterializedViews(String tenantId) {
+    for (String matViewName : materializedViewNames) {
+      String fullName = tenantId + "_mod_fqm_manager." + matViewName;
+      log.info("Refreshing materialized view {}", fullName);
+      jooqContext.execute(REFRESH_MATERIALIZED_VIEW_SQL + fullName);
+    }
+  }
+}

src/main/java/org/folio/fqm/repository/QueryRepository.java

@@ -73,10 +73,10 @@ public Optional<Query> getQuery(UUID queryId, boolean useCache) {
       .fetchOneInto(Query.class));
   }
 
-  public List<UUID> getQueryIdsCompletedBefore(Duration duration) {
+  public List<UUID> getQueryIdsStartedBefore(Duration duration) {
     return jooqContext.select(field(QUERY_ID))
       .from(table(QUERY_DETAILS_TABLE))
-      .where(field("end_date").
+      .where(field("start_date").
         lessOrEqual(OffsetDateTime.now().minus(duration)))
       .fetchInto(UUID.class);
   }

src/main/java/org/folio/fqm/resource/MaterializedViewRefreshController.java

@@ -0,0 +1,21 @@
+package org.folio.fqm.resource;
+
+import lombok.RequiredArgsConstructor;
+import org.folio.fqm.service.MaterializedViewRefreshService;
+import org.folio.spring.FolioExecutionContext;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequiredArgsConstructor
+public class MaterializedViewRefreshController implements MaterializedViewsApi {
+  private final FolioExecutionContext executionContext;
+  private final MaterializedViewRefreshService materializedViewRefreshService;
+
+  @Override
+  public ResponseEntity<Void> refreshMaterializedViews() {
+    materializedViewRefreshService.refreshMaterializedViews(executionContext.getTenantId());
+    return new ResponseEntity<>(HttpStatus.NO_CONTENT);
+  }
+}

src/main/java/org/folio/fqm/service/MaterializedViewRefreshService.java

@@ -0,0 +1,15 @@
+package org.folio.fqm.service;
+
+import lombok.RequiredArgsConstructor;
+import org.folio.fqm.repository.MaterializedViewRefreshRepository;
+import org.springframework.stereotype.Service;
+
+@Service
+@RequiredArgsConstructor
+public class MaterializedViewRefreshService {
+  private final MaterializedViewRefreshRepository materializedViewRefreshRepository;
+
+  public void refreshMaterializedViews(String tenantId) {
+    materializedViewRefreshRepository.refreshMaterializedViews(tenantId);
+  }
+}

src/main/java/org/folio/fqm/service/QueryManagementService.java

@@ -133,7 +133,7 @@ public Optional<QueryDetails> getQuery(UUID queryId, boolean includeResults, int
    */
   @Transactional
   public PurgedQueries deleteOldQueries() {
-    List<UUID> queryIds = queryRepository.getQueryIdsCompletedBefore(queryRetentionDuration);
+    List<UUID> queryIds = queryRepository.getQueryIdsStartedBefore(queryRetentionDuration);
     log.info("Deleting the queries with queryIds {}", queryIds);
     deleteQueryAndResults(queryIds);
     return new PurgedQueries().deletedQueryIds(queryIds);

src/main/resources/db/changelog/changelog-master.xml

@@ -6,5 +6,7 @@
 
   <include file="changes/v1.0.0/changelog-v1.0.0.xml" relativeToChangelogFile="true"/>
   <include file="changes/v1.0.1/changelog-v1.0.1.xml" relativeToChangelogFile="true"/>
+  <include file="changes/v1.0.2/changelog-v1.0.2.xml" relativeToChangelogFile="true"/>
+  <include file="changes/v1.0.3/changelog-v1.0.3.xml" relativeToChangelogFile="true"/>
 
 </databaseChangeLog>

src/main/resources/db/changelog/changes/v1.0.2/changelog-v1.0.2.xml

@@ -0,0 +1,16 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                   http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
+
+  <include file="remove_items_instance_title_searchable.xml" relativeToChangelogFile="true"/>
+  <include file="update_item_details_entity_type_definition.xml" relativeToChangelogFile="true"/>
+
+  <include file="remove_items_holdingsrecord_instance_title_searchable.xml" relativeToChangelogFile="true"/>
+  <include file="update_item_holdingsrecord_entity_type_definition.xml" relativeToChangelogFile="true"/>
+
+  <include file="fix_user_details_preferred_contact_type.xml" relativeToChangelogFile="true"/>
+  <include file="update_user_details_entity_type_definition.xml" relativeToChangelogFile="true"/>
+
+</databaseChangeLog>

src/main/resources/db/changelog/changes/v1.0.2/fix_user_details_preferred_contact_type.xml

@@ -0,0 +1,98 @@
+<databaseChangeLog xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                   http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
+  <changeSet id="create-view-user-details" author="[email protected]">
+    <!-- ensure other schemas/tables exist (primarily to prevent invalid references in integration tests) -->
+    <preConditions onFail="CONTINUE">
+      <viewExists viewName="src_users_groups"/>
+      <viewExists viewName="src_users_users"/>
+    </preConditions>
+    <createView
+      replaceIfExists="true"
+      viewName="drv_user_details">
+      SELECT
+      userDetails.jsonb -> 'personal' ->> 'firstName' as user_first_name,
+      userDetails.jsonb -> 'personal' ->> 'lastName' as user_last_name,
+      userDetails.jsonb ->>'barcode' as user_barcode,
+      userDetails.jsonb ->>'username' as username,
+      userDetails.id as id,
+      userDetails.jsonb ->> 'externalSystemId' as user_external_system_id,
+      userDetails.jsonb ->> 'active' as user_active,
+      userDetails.jsonb -> 'personal' ->> 'email' as user_email,
+      userDetails.jsonb -> 'metadata' ->> 'createdDate' as user_created_date,
+      userDetails.jsonb -> 'metadata' ->> 'updatedDate' as user_updated_date,
+      userDetails.jsonb -> 'personal' ->> 'preferredFirstName' as user_preferred_first_name,
+      userDetails.jsonb -> 'personal' ->> 'middleName' as user_middle_name,
+      userDetails.jsonb -> 'personal' ->> 'phone' as user_phone,
+      userDetails.jsonb -> 'personal' ->> 'mobilePhone' as user_mobile_phone,
+      userDetails.jsonb -> 'personal' ->> 'dateOfBirth' as user_date_of_birth,
+      userDetails.jsonb ->> 'expirationDate'::text AS user_expiration_date,
+      userDetails.jsonb ->> 'enrollmentDate'::text AS user_enrollment_date,
+      patron_id_ref_data.jsonb ->> 'group'::text AS user_patron_group,
+      patron_id_ref_data.id::text AS user_patron_group_id,
+      UserDetails.jsonb -> 'personal' ->> 'preferredContactTypeId' as user_preferred_contact_type_id,
+      CASE UserDetails.jsonb -> 'personal' ->> 'preferredContactTypeId'
+      WHEN '001' THEN 'Mail (Primary Address)'
+      WHEN '002' THEN 'Email'
+      WHEN '003' THEN 'Text Message'
+      ELSE 'unknown'
+      END AS user_preferred_contact_type,
+      concat_ws(', '::text,
+      NULLIF(( SELECT subquery.addressline1
+      FROM ( SELECT add_id.value ->> 'addressLine1'::text AS addressline1,
+      row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
+      WHERE subquery.row_num = 1), ''::text),
+      NULLIF(( SELECT subquery.addressline2
+      FROM ( SELECT add_id.value ->> 'addressLine2'::text AS addressline2,
+      row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
+      WHERE subquery.row_num = 1), ''::text),
+      NULLIF(( SELECT subquery.city
+      FROM ( SELECT add_id.value ->> 'city'::text AS city,
+      row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
+      WHERE subquery.row_num = 1), ''::text),
+      NULLIF(( SELECT subquery.region
+      FROM ( SELECT add_id.value ->> 'region'::text AS region,
+      row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
+      WHERE subquery.row_num = 1), ''::text),
+      NULLIF(( SELECT subquery.postalcode
+      FROM ( SELECT add_id.value ->> 'postalCode'::text AS postalcode,
+      row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
+      WHERE subquery.row_num = 1), ''::text),
+      NULLIF(( SELECT subquery.countryid
+      FROM ( SELECT add_id.value ->> 'countryId'::text AS countryid,
+      row_number() OVER (ORDER BY (add_id.value ->> 'primaryAddress'::text)) AS row_num
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) subquery
+      WHERE subquery.row_num = 1), ''::text)) AS user_primary_address,
+      ( SELECT array_agg(add_id.value ->> 'city'::text) FILTER (WHERE (add_id.value ->> 'city'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS cities,
+      ( SELECT array_agg(add_id.value ->> 'region'::text) FILTER (WHERE (add_id.value ->> 'region'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS regions,
+      ( SELECT array_agg(add_id.value ->> 'countryId'::text) FILTER (WHERE (add_id.value ->> 'countryId'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS country_ids,
+      ( SELECT array_agg(add_id.value ->> 'postalCode'::text) FILTER (WHERE (add_id.value ->> 'postalCode'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS postal_codes,
+      ( SELECT array_agg(add_id.value ->> 'addressLine1'::text) FILTER (WHERE (add_id.value ->> 'addressLine1'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS address_line1,
+      ( SELECT array_agg(add_id.value ->> 'addressLine2'::text) FILTER (WHERE (add_id.value ->> 'addressLine2'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS address_line2,
+      ( SELECT array_agg(add_id.value ->> 'addressTypeId'::text) FILTER (WHERE (add_id.value ->> 'addressTypeId'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)) AS address_ids,
+      ( SELECT array_agg(a.jsonb ->> 'addressType'::text) FILTER (WHERE (a.jsonb ->> 'addressType'::text) IS NOT NULL) AS array_agg
+      FROM jsonb_array_elements((userdetails.jsonb -> 'personal'::text) -> 'addresses'::text) add_id(value)
+      JOIN src_users_addresstype a ON (add_id.value ->> 'addressTypeId'::text) = a.id::text) AS address_type_names,
+      array_agg(temp_departments.id::text) FILTER (where temp_departments.id is not null) as department_ids,
+      array_agg(temp_departments.jsonb ->> 'name') FILTER (where temp_departments.jsonb ->> 'name' is not null) as department_names
+      FROM src_users_users userdetails
+      LEFT JOIN src_users_groups patron_id_ref_data ON patron_id_ref_data.id = ((userdetails.jsonb ->> 'patronGroup'::text)::uuid)
+      LEFT JOIN src_users_departments as temp_departments ON userdetails.jsonb -> 'departments' ?? temp_departments.id::text
+      GROUP BY userdetails.id, userdetails.jsonb, patron_id_ref_data.id, patron_id_ref_data.jsonb
+    </createView>
+  </changeSet>
+
+</databaseChangeLog>