Skip to content

Commit

Permalink
feat(reindex): implement new re-index flow for instance records
Browse files Browse the repository at this point in the history
feat(reindex): implement new re-index flow for instance records

Closes: MSEARCH-793, MSEARCH-794, MSEARCH-796, MSEARCH-797, MSEARCH-798, MSEARCH-799, MSEARCH-800, MSEARCH-801, MSEARCH-802
  • Loading branch information
psmagin authored Oct 8, 2024
2 parents 7692bee + 8a696d5 commit 59e1457
Show file tree
Hide file tree
Showing 351 changed files with 9,309 additions and 6,769 deletions.
13 changes: 7 additions & 6 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
## v3.3.0 YYYY-mm-DD
### Breaking changes
* Description ([ISSUE_NUMBER](https://folio-org.atlassian.net/browse/ISSUE_NUMBER))
## v4.0.0 YYYY-mm-DD

### New APIs versions
* Provides `indices v0.7`
* Provides `indices v0.8`
* Provides `search v1.3`
* Requires `locations v3.0`
* Provides `consortium-search v1.2`
* Requires `instance-storage v10.0 or v11.0`
* Provides `browse v1.4`
* Requires `locations v3.0`
* Requires `instance-storage v10.3 or v11.0`

### Features
* Implement indices recreation of the Linked Data resources ([MSEARCH-820](https://issues.folio.org/browse/MSEARCH-820))
Expand Down Expand Up @@ -39,6 +38,8 @@
* Implement Reindexing of Libraries ([MSEARCH-766](https://issues.folio.org/browse/MSEARCH-766))
* Implement Reindexing of Institutions ([MSEARCH-768](https://issues.folio.org/browse/MSEARCH-768))
* Create computed field for sorting and filtering Date 1 ([MSEARCH-806](https://folio-org.atlassian.net/browse/MSEARCH-806))
* Support filters for subject source and type on subject browse ([MSEARCH-805](https://folio-org.atlassian.net/browse/MSEARCH-805))
* Implement new re-index flow for instance records ([MSEARCH-793](https://folio-org.atlassian.net/issues/MSEARCH-793), [MSEARCH-794](https://folio-org.atlassian.net/issues/MSEARCH-794), [MSEARCH-796](https://folio-org.atlassian.net/issues/MSEARCH-796), [MSEARCH-797](https://folio-org.atlassian.net/issues/MSEARCH-797), [MSEARCH-798](https://folio-org.atlassian.net/issues/MSEARCH-798), [MSEARCH-799](https://folio-org.atlassian.net/issues/MSEARCH-799), [MSEARCH-800](https://folio-org.atlassian.net/issues/MSEARCH-800), [MSEARCH-801](https://folio-org.atlassian.net/issues/MSEARCH-801), [MSEARCH-802](https://folio-org.atlassian.net/issues/MSEARCH-802))

### Bug fixes
* Do not delete kafka topics if collection topic is enabled ([MSEARCH-725](https://folio-org.atlassian.net/browse/MSEARCH-725))
Expand Down
135 changes: 121 additions & 14 deletions README.md

Large diffs are not rendered by default.

199 changes: 135 additions & 64 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"provides": [
{
"id": "indices",
"version": "0.7",
"version": "0.8",
"handlers": [
{
"methods": [
Expand Down Expand Up @@ -74,6 +74,43 @@
"modulePermissions": [
"user-tenants.collection.get"
]
},
{
"methods": [
"GET"
],
"pathPattern": "/search/index/instance-records/reindex/status",
"permissionsRequired": [
"search.index.instance-records.reindex.status.get"
]
},
{
"methods": [
"POST"
],
"pathPattern": "/search/index/instance-records/reindex/full",
"permissionsRequired": [
"search.index.instance-records.reindex.full.post"
],
"modulePermissions": [
"inventory-storage.reindex-records.publish.post",
"inventory-storage.instances.collection.get",
"inventory-storage.items.collection.get",
"inventory-storage.holdings.collection.get",
"user-tenants.collection.get"
]
},
{
"methods": [
"POST"
],
"pathPattern": "/search/index/instance-records/reindex/upload",
"permissionsRequired": [
"search.index.instance-records.reindex.upload.post"
],
"modulePermissions": [
"user-tenants.collection.get"
]
}
]
},
Expand Down Expand Up @@ -349,7 +386,7 @@
},
{
"id": "browse",
"version": "1.3",
"version": "1.4",
"handlers": [
{
"methods": [
Expand Down Expand Up @@ -551,7 +588,11 @@
"inventory-storage.locations.collection.get",
"inventory-storage.location-units.campuses.collection.get",
"inventory-storage.location-units.libraries.collection.get",
"inventory-storage.location-units.institutions.collection.get"
"inventory-storage.location-units.institutions.collection.get",
"inventory-storage.reindex-records.publish.post",
"inventory-storage.instances.collection.get",
"inventory-storage.items.collection.get",
"inventory-storage.holdings.collection.get"
]
},
{
Expand All @@ -568,12 +609,20 @@
{
"id": "user-tenants",
"version": "1.0"
},
{
"id": "consortia",
"version": "1.0"
},
{
"id": "inventory-reindex-records",
"version": "1.0"
}
],
"requires": [
{
"id": "instance-storage",
"version": "10.0 11.0"
"version": "10.3 11.0"
},
{
"id": "login",
Expand Down Expand Up @@ -641,6 +690,16 @@
"displayName": "Search - updates settings for the index",
"description": "Updates settings for the index"
},
{
"permissionName": "search.index.instance-records.reindex.full.post",
"displayName": "Search - starts inventory instance records full reindex operation",
"description": "Starts inventory instance records reindex operation"
},
{
"permissionName": "search.index.instance-records.reindex.upload.post",
"displayName": "Search - starts inventory instance records reindex upload operation",
"description": "Starts inventory instance records reindex upload operation"
},
{
"permissionName": "search.instances.collection.get",
"displayName": "Search - searches instances by given query",
Expand Down Expand Up @@ -716,6 +775,11 @@
"displayName": "Search - starts inventory reindex operation",
"description": "Starts inventory reindex operation"
},
{
"permissionName": "search.index.instance-records.reindex.status.get",
"displayName": "Search - returns reindex status for entities",
"description": "Returns reindex status for entities"
},
{
"permissionName": "search.facets.collection.get",
"displayName": "Search - returns facets for a query for given filter options by record type",
Expand Down Expand Up @@ -818,19 +882,46 @@
"permissionName": "consortium-search.institutions.collection.get",
"displayName": "Consortium Search - fetch institutions records",
"description": "Returns institution records in consortium"
},
{
"permissionName": "search.index.instance-records.reindex.all",
"displayName": "Search Instance Records Re-Index - All permissions",
"description": "Entire set of permissions needed to manage Instance Records Re-Index",
"subPermissions": [
"search.index.instance-records.reindex.status.get",
"search.index.instance-records.reindex.full.post",
"search.index.instance-records.reindex.upload.post"
]
}

],
"metadata": {
"user": {
"type": "system",
"permissions": [
"user-tenants.collection.get",
"users.collection.get",
"users.item.post",
"login.item.post",
"login.item.delete",
"perms.users.get",
"perms.users.item.post",
"perms.users.assign.immutable",
"perms.users.assign.mutable",
"inventory-storage.instance.reindex.post",
"authority-storage.authority.reindex.post",
"inventory-storage.inventory-view.instances.collection.get",
"inventory-storage.identifier-types.collection.get",
"inventory-storage.alternative-title-types.collection.get",
"inventory-storage.call-number-types.collection.get",
"inventory-storage.classification-types.collection.get",
"users.collection.get",
"user-tenants.collection.get"
"instance-authority-links.authorities.bulk.post",
"inventory-storage.locations.collection.get",
"inventory-storage.location-units.campuses.collection.get",
"inventory-storage.location-units.libraries.collection.get",
"inventory-storage.location-units.institutions.collection.get",
"inventory-storage.reindex-records.publish.post",
"inventory-storage.instances.collection.get",
"inventory-storage.items.collection.get",
"inventory-storage.holdings.collection.get"
]
}
},
Expand Down Expand Up @@ -915,59 +1006,24 @@
"description": "Custom number of kafka concurrent threads for authority message consuming."
},
{
"name": "KAFKA_CONTRIBUTORS_CONCURRENCY",
"value": "2",
"description": "Custom number of kafka concurrent threads for contributor message consuming."
},
{
"name": "KAFKA_CONTRIBUTORS_CONSUMER_PATTERN",
"value": "(folio\\.)(.*\\.)search\\.instance-contributor",
"description": "Custom subscription pattern for Kafka contributor message consumers."
},
{
"name": "KAFKA_CONTRIBUTORS_TOPIC_PARTITIONS",
"value": "50",
"description": "Amount of partitions for `search.instance-contributor` topic."
},
{
"name": "KAFKA_CONTRIBUTORS_TOPIC_REPLICATION_FACTOR",
"value": "",
"description": "Replication factor for `search.instance-contributor` topic."
},
{
"name": "KAFKA_CONSORTIUM_INSTANCE_CONCURRENCY",
"value": "2",
"description": "Custom number of kafka concurrent threads for consortium.instance message consuming."
"name": "KAFKA_REINDEX_RANGE_INDEX_CONCURRENCY",
"value": "1",
"description": "Custom number of kafka concurrent threads for `search.reindex.range-index` message consuming."
},
{
"name": "KAFKA_CONSORTIUM_INSTANCE_TOPIC_PARTITIONS",
"value": "50",
"description": "Amount of partitions for `search.consortium.instance` topic."
"name": "KAFKA_REINDEX_RANGE_INDEX_TOPIC_PARTITIONS",
"value": "16",
"description": "Amount of partitions for `search.reindex.range-index` topic."
},
{
"name": "KAFKA_CONSORTIUM_INSTANCE_TOPIC_REPLICATION_FACTOR",
"name": "KAFKA_REINDEX_RANGE_INDEX_TOPIC_REPLICATION_FACTOR",
"value": "",
"description": "Replication factor for `search.consortium.instance` topic."
"description": "Replication factor for `search.reindex.range-index` topic."
},
{
"name": "KAFKA_SUBJECTS_CONCURRENCY",
"name": "KAFKA_REINDEX_RECORDS_CONCURRENCY",
"value": "2",
"description": "Custom number of kafka concurrent threads for subject message consuming."
},
{
"name": "KAFKA_SUBJECTS_CONSUMER_PATTERN",
"value": "(folio\\.)(.*\\.)search\\.instance-subject",
"description": "Custom subscription pattern for Kafka subject message consumers."
},
{
"name": "KAFKA_SUBJECTS_TOPIC_PARTITIONS",
"value": "50",
"description": "Amount of partitions for `search.instance-subject` topic."
},
{
"name": "KAFKA_SUBJECTS_TOPIC_REPLICATION_FACTOR",
"value": "",
"description": "Replication factor for `search.instance-subject` topic."
"description": "Custom number of kafka concurrent threads for `inventory.reindex-records` message consuming."
},
{
"name": "KAFKA_CONSUMER_MAX_POLL_RECORDS",
Expand Down Expand Up @@ -1041,16 +1097,6 @@
"name": "DB_MAXPOOLSIZE",
"value": "5"
},
{
"name": "INSTANCE_SUBJECTS_INDEXING_RETRY_ATTEMPTS",
"value": "3",
"description": "Amount of retry attempts to delete instance subject resources."
},
{
"name": "INSTANCE_CONTRIBUTORS_INDEXING_RETRY_ATTEMPTS",
"value": "3",
"description": "Amount of retry attempts to delete instance contributor resources."
},
{
"name": "INDEXING_DATA_FORMAT",
"value": "smile",
Expand Down Expand Up @@ -1147,9 +1193,34 @@
"description": "Defines number of locations to retrieve per inventory http request on locations reindex process"
},
{
"name": "MAX_SEARCH_BATCH_REQUEST_IDS_COUNT,",
"name": "MAX_SEARCH_BATCH_REQUEST_IDS_COUNT",
"value": "20_000",
"description": "Defines maximum batch request IDs count for searching consolidated items/holdings in consortium"
},
{
"name": "REINDEX_MERGE_RANGE_SIZE",
"value": "500",
"description": "The range size that represents the number of merge entities to process during the Merge process of reindex"
},
{
"name": "REINDEX_UPLOAD_RANGE_SIZE",
"value": "1_000",
"description": "The range size that represents the number of upload entities to process during the Upload process of reindex"
},
{
"name": "REINDEX_UPLOAD_RANGE_LEVEL",
"value": "3",
"description": "The level of deepness of upload range generator affecting the number of ranges to be generated"
},
{
"name": "REINDEX_MERGE_RANGE_PUBLISHER_CORE_POOL_SIZE",
"value": "3",
"description": "The number of threads for publishing the merge ranges to keep in the pool, even if they are idle."
},
{
"name": "REINDEX_MERGE_RANGE_PUBLISHER_MAX_POOL_SIZE",
"value": "6",
"description": "The maximum number of threads for publishing the merge ranges to allow in the pool."
}
]
}
Expand Down
1 change: 1 addition & 0 deletions docker/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ services:
DB_HOST: ${DB_HOST}
DB_DATABASE: ${DB_DATABASE}
DB_PASSWORD: ${DB_PASSWORD}
OKAPI_URL: http://api-mock:8080
JAVA_OPTIONS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:${DEBUG_PORT}"

api-mock:
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<name>mod-search</name>
<groupId>org.folio</groupId>
<artifactId>mod-search</artifactId>
<version>3.3.0-SNAPSHOT</version>
<version>4.0.0-SNAPSHOT</version>
<description>FOLIO search service</description>
<packaging>jar</packaging>

Expand Down
26 changes: 26 additions & 0 deletions src/main/java/org/folio/search/client/ConsortiumTenantsClient.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package org.folio.search.client;

import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;

import java.util.List;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestParam;

@FeignClient("consortia")
public interface ConsortiumTenantsClient {

/**
* Get tenants by consortium id.
*
* @return consortium tenants if executed under consortium central 'tenantId' context
* */
@GetMapping(value = "/{consortiumId}/tenants", produces = APPLICATION_JSON_VALUE)
ConsortiumTenants getConsortiumTenants(@PathVariable("consortiumId") String consortiumId,
@RequestParam("limit") int limit);

record ConsortiumTenants(List<ConsortiumTenant> tenants) { }

record ConsortiumTenant(String id, boolean isCentral) { }
}
22 changes: 22 additions & 0 deletions src/main/java/org/folio/search/client/InventoryInstanceClient.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package org.folio.search.client;

import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;

import java.net.URI;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;

@FeignClient("instance-storage")
public interface InventoryInstanceClient {

/**
* Retrieves Inventory Records count with given URI.
*
* @param uri URI to retrieve count of inventory record
* @return count represented as {@link InventoryRecordsCountDto}
*/
@GetMapping(produces = APPLICATION_JSON_VALUE)
InventoryRecordsCountDto getInventoryRecordsCount(URI uri);

record InventoryRecordsCountDto(int totalRecords) { }
}
Loading

0 comments on commit 59e1457

Please sign in to comment.