Skip to content

Commit

Permalink
Merge branch 'master' into msearch-785
Browse files Browse the repository at this point in the history
# Conflicts:
#	NEWS.md
#	README.md
  • Loading branch information
mukhiddin-yusuf committed Jun 27, 2024
2 parents 7a78d1a + 7c1baac commit b83a267
Show file tree
Hide file tree
Showing 55 changed files with 1,947 additions and 146 deletions.
5 changes: 5 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,13 @@
* Remove ability to match on LCCN searches without a prefix ([MSEARCH-752](https://folio-org.atlassian.net/browse/MSEARCH-752))
* Search consolidated items/holdings data in consortium ([MSEARCH-759](https://folio-org.atlassian.net/browse/MSEARCH-759))
* Create bibframe index and process bibframe events ([MSEARCH-781](https://folio-org.atlassian.net/browse/MSEARCH-781))
* Create bibframe authority index and process bibframe authority events ([MSEARCH-784](https://folio-org.atlassian.net/browse/MSEARCH-784))
* Allow Unified List of Inventory Locations in a Consortium to be fetched by member tenants ([MSEARCH-660](https://folio-org.atlassian.net/browse/MSEARCH-660))
* Implement Indexing of Campuses from Kafka ([MSEARCH-770](https://issues.folio.org/browse/MSEARCH-770))
* Extend response with additional Location fields for Inventory Locations in a Consortium endpoint ([MSEARCH-775](https://folio-org.atlassian.net/browse/MSEARCH-775))
* Implement Indexing of Institutions from Kafka ([MSEARCH-771](https://issues.folio.org/browse/MSEARCH-771))
* Implement Indexing of Libraries from Kafka ([MSEARCH-769](https://issues.folio.org/browse/MSEARCH-769))
* Return Unified List of Inventory Campuses in a Consortium ([MSEARCH-773](https://issues.folio.org/browse/MSEARCH-773))
* Increase batch IDs limit for search consolidated items/holdings in consortium ([MSEARCH-785](https://folio-org.atlassian.net/browse/MSEARCH-785))

### Bug fixes
Expand All @@ -31,6 +35,7 @@
* Browse: Duplicate results in exact match with diacritics ([MSEARCH-751](https://folio-org.atlassian.net/browse/MSEARCH-751))
* Classification browse: Fix instances count for Shared Facet ([MSEARCH-761](https://folio-org.atlassian.net/browse/MSEARCH-761))
* Subjects/Contributors browse: Fix instances count for Shared Facet ([MSEARCH-782](https://folio-org.atlassian.net/browse/MSEARCH-782))
* Search Resources IDs: Local instances are not searchable with requests from member tenants ([MSEARCH-762](https://folio-org.atlassian.net/browse/MSEARCH-762))

### Tech Dept
* Re-Index: delete all records from consortium_instance on full re-index ([MSEARCH-744](https://folio-org.atlassian.net/browse/MSEARCH-744))
Expand Down
142 changes: 72 additions & 70 deletions README.md

Large diffs are not rendered by default.

26 changes: 26 additions & 0 deletions descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,15 @@
"search.bibframe.collection.get"
]
},
{
"methods": [
"GET"
],
"pathPattern": "/search/bibframe/authorities",
"permissionsRequired": [
"search.bibframe.authority.collection.get"
]
},
{
"methods": [
"GET"
Expand Down Expand Up @@ -189,6 +198,18 @@
"user-tenants.collection.get"
]
},
{
"methods": [
"GET"
],
"pathPattern": "/search/consortium/campuses",
"permissionsRequired": [
"consortium-search.campuses.collection.get"
],
"modulePermissions": [
"user-tenants.collection.get"
]
},
{
"methods": [
"GET"
Expand Down Expand Up @@ -724,6 +745,11 @@
"permissionName": "consortium-search.locations.collection.get",
"displayName": "Consortium Search - fetch locations records",
"description": "Returns location records in consortium"
},
{
"permissionName": "consortium-search.campuses.collection.get",
"displayName": "Consortium Search - fetch campuses records",
"description": "Returns campus records in consortium"
}
],
"launchDescriptor": {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.search.domain.dto.BatchIdsDto;
import org.folio.search.domain.dto.ConsortiumCampusCollection;
import org.folio.search.domain.dto.ConsortiumHolding;
import org.folio.search.domain.dto.ConsortiumHoldingCollection;
import org.folio.search.domain.dto.ConsortiumItem;
Expand All @@ -20,6 +21,7 @@
import org.folio.search.model.service.CqlSearchRequest;
import org.folio.search.model.types.ResourceType;
import org.folio.search.rest.resource.SearchConsortiumApi;
import org.folio.search.service.consortium.ConsortiumCampusService;
import org.folio.search.service.consortium.ConsortiumInstanceSearchService;
import org.folio.search.service.consortium.ConsortiumInstanceService;
import org.folio.search.service.consortium.ConsortiumLocationService;
Expand All @@ -44,6 +46,7 @@ public class SearchConsortiumController implements SearchConsortiumApi {
private final ConsortiumInstanceService instanceService;
private final ConsortiumLocationService locationService;
private final ConsortiumInstanceSearchService searchService;
private final ConsortiumCampusService campusService;

@Override
public ResponseEntity<ConsortiumHoldingCollection> getConsortiumHoldings(String tenantHeader, String instanceId,
Expand Down Expand Up @@ -95,6 +98,21 @@ public ResponseEntity<ConsortiumLocationCollection> getConsortiumLocations(Strin
.totalRecords(result.getTotalRecords()));
}

@Override
public ResponseEntity<ConsortiumCampusCollection> getConsortiumCampuses(String tenantHeader,
String tenantId,
Integer limit,
Integer offset,
String sortBy,
SortOrder sortOrder) {
var result = campusService.fetchCampuses(tenantHeader, tenantId, limit, offset, sortBy, sortOrder);

return ResponseEntity.ok(new
ConsortiumCampusCollection()
.campuses(result.getRecords())
.totalRecords(result.getTotalRecords()));
}

@Override
public ResponseEntity<ConsortiumHolding> getConsortiumHolding(UUID id, String tenantHeader) {
var tenant = verifyAndGetTenant(tenantHeader);
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/org/folio/search/controller/SearchController.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import org.folio.search.domain.dto.Authority;
import org.folio.search.domain.dto.AuthoritySearchResult;
import org.folio.search.domain.dto.Bibframe;
import org.folio.search.domain.dto.BibframeAuthority;
import org.folio.search.domain.dto.BibframeSearchAuthorityResult;
import org.folio.search.domain.dto.BibframeSearchResult;
import org.folio.search.domain.dto.Instance;
import org.folio.search.domain.dto.InstanceSearchResult;
Expand Down Expand Up @@ -65,6 +67,23 @@ public ResponseEntity<BibframeSearchResult> searchBibframe(String tenant, String
);
}

@Override
public ResponseEntity<BibframeSearchAuthorityResult> searchBibframeAuthorities(String tenant,
String query,
Integer limit,
Integer offset) {
var searchRequest = CqlSearchRequest.of(
BibframeAuthority.class, tenant, query, limit, offset, true);
var result = searchService.search(searchRequest);
return ResponseEntity.ok(new BibframeSearchAuthorityResult()
.searchQuery(query)
.content(result.getRecords())
.pageNumber(divPlusOneIfRemainder(offset, limit))
.totalPages(divPlusOneIfRemainder(result.getTotalRecords(), limit))
.totalRecords(result.getTotalRecords())
);
}

private int divPlusOneIfRemainder(int one, int two) {
var modulo = one % two;
return one / two + (modulo > 0 ? 1 : 0);
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/org/folio/search/cql/CqlSearchQueryConverter.java
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,22 @@ public SearchSourceBuilder convertForConsortia(String query, String resource) {
return convertForConsortia(query, resource, false);
}

/**
* Converts given CQL search query value to the elasticsearch {@link SearchSourceBuilder} object.
* Wraps base 'convert' and adds active affiliation tenantId filter in case of consortia mode
*
* @param query cql query to parse
* @param resource resource name
* @param tenantId active affiliation member tenant name
* @return search source as {@link SearchSourceBuilder} object with query and sorting conditions
*/
public SearchSourceBuilder convertForConsortia(String query, String resource, String tenantId) {
var sourceBuilder = convert(query, resource);
var queryBuilder = consortiumSearchHelper
.filterQueryForActiveAffiliation(sourceBuilder.query(), resource, tenantId);
return sourceBuilder.query(queryBuilder);
}

public SearchSourceBuilder convertForConsortia(String query, String resource, boolean consortiumConsolidated) {
var sourceBuilder = convert(query, resource);
if (consortiumConsolidated) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import static org.folio.search.utils.SearchConverterUtils.getEventPayload;
import static org.folio.search.utils.SearchConverterUtils.getResourceEventId;
import static org.folio.search.utils.SearchConverterUtils.getResourceSource;
import static org.folio.search.utils.SearchUtils.BIBFRAME_RESOURCE;
import static org.folio.search.utils.SearchUtils.ID_FIELD;
import static org.folio.search.utils.SearchUtils.INSTANCE_ID_FIELD;
import static org.folio.search.utils.SearchUtils.INSTANCE_RESOURCE;
Expand Down Expand Up @@ -206,7 +205,7 @@ public void handleBibframeEvents(List<ConsumerRecord<String, ResourceEvent>> con
log.info("Processing bibframe events from Kafka [number of events: {}]", consumerRecords.size());
var batch = consumerRecords.stream()
.map(ConsumerRecord::value)
.map(bibframe -> bibframe.resourceName(BIBFRAME_RESOURCE).id(getResourceEventId(bibframe)))
.map(bibframe -> bibframe.id(getResourceEventId(bibframe)))
.toList();

indexResources(batch, resourceService::indexResources);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,21 @@
@Component
public class ResourceEventBatchInterceptor implements BatchInterceptor<String, ResourceEvent> {

private static final Map<String, String> TOPIC_TO_RESOURCE_MAP = Map.of(
"inventory.instance", SearchUtils.INSTANCE_RESOURCE,
"inventory.holdings-record", SearchUtils.INSTANCE_RESOURCE,
"inventory.item", SearchUtils.INSTANCE_RESOURCE,
"inventory.bound-with", SearchUtils.INSTANCE_RESOURCE,
"authorities.authority", SearchUtils.AUTHORITY_RESOURCE,
"search.instance-contributor", SearchUtils.CONTRIBUTOR_RESOURCE,
"search.instance-subject", SearchUtils.INSTANCE_SUBJECT_RESOURCE,
"inventory.classification-type", SearchUtils.CLASSIFICATION_TYPE_RESOURCE,
"inventory.location", SearchUtils.LOCATION_RESOURCE,
"inventory.campus", SearchUtils.CAMPUS_RESOURCE
private static final Map<String, String> TOPIC_TO_RESOURCE_MAP = Map.ofEntries(
Map.entry("inventory.instance", SearchUtils.INSTANCE_RESOURCE),
Map.entry("inventory.holdings-record", SearchUtils.INSTANCE_RESOURCE),
Map.entry("inventory.item", SearchUtils.INSTANCE_RESOURCE),
Map.entry("inventory.bound-with", SearchUtils.INSTANCE_RESOURCE),
Map.entry("authorities.authority", SearchUtils.AUTHORITY_RESOURCE),
Map.entry("search.instance-contributor", SearchUtils.CONTRIBUTOR_RESOURCE),
Map.entry("search.instance-subject", SearchUtils.INSTANCE_SUBJECT_RESOURCE),
Map.entry("inventory.classification-type", SearchUtils.CLASSIFICATION_TYPE_RESOURCE),
Map.entry("inventory.location", SearchUtils.LOCATION_RESOURCE),
Map.entry("inventory.campus", SearchUtils.CAMPUS_RESOURCE),
Map.entry("inventory.institution", SearchUtils.INSTITUTION_RESOURCE),
Map.entry("inventory.library", SearchUtils.LIBRARY_RESOURCE),
Map.entry("search.bibframe", SearchUtils.BIBFRAME_RESOURCE),
Map.entry("search.bibframe-authorities", SearchUtils.BIBFRAME_AUTHORITY_RESOURCE)
);

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import lombok.Data;
import lombok.With;
import lombok.extern.jackson.Jacksonized;
import org.folio.search.domain.dto.Metadata;

/**
* Describes Campus object that comes from external channels.
Expand All @@ -25,5 +26,7 @@ public class CampusDto {
private String code;
@JsonProperty("institutionId")
private String institutionId;
@JsonProperty("metadata")
private Metadata metadata;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package org.folio.search.model.dto.locationunit;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import lombok.With;
import lombok.extern.jackson.Jacksonized;
import org.folio.search.domain.dto.Metadata;

/**
* Describes Institution object that comes from external channels.
*/
@Data
@With
@Builder
@Jacksonized
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public class InstitutionDto {

@JsonProperty("id")
private String id;
@JsonProperty("name")
private String name;
@JsonProperty("code")
private String code;
@JsonProperty("metadata")
private Metadata metadata;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package org.folio.search.model.dto.locationunit;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Builder;
import lombok.Data;
import lombok.With;
import lombok.extern.jackson.Jacksonized;
import org.folio.search.domain.dto.Metadata;

/**
* Describes Campus object that comes from external channels.
*/
@Data
@With
@Builder
@Jacksonized
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public class LibraryDto {

@JsonProperty("id")
private String id;
@JsonProperty("name")
private String name;
@JsonProperty("code")
private String code;
@JsonProperty("campusId")
private String campusId;
@JsonProperty("metadata")
private Metadata metadata;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package org.folio.search.repository;

import static org.folio.search.utils.SearchUtils.TENANT_ID_FIELD_NAME;
import static org.folio.search.utils.SearchUtils.performExceptionalOperation;
import static org.opensearch.search.sort.SortOrder.ASC;
import static org.opensearch.search.sort.SortOrder.DESC;

import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.search.domain.dto.ConsortiumCampus;
import org.folio.search.domain.dto.SortOrder;
import org.folio.search.model.SearchResult;
import org.folio.search.service.converter.ElasticsearchDocumentConverter;
import org.jetbrains.annotations.NotNull;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.RestHighLevelClient;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.search.sort.SortBuilders;
import org.springframework.stereotype.Repository;

@Log4j2
@Repository
@RequiredArgsConstructor
public class ConsortiumCampusRepository {

public static final String CAMPUS_INDEX = "campus";
private static final String OPERATION_TYPE = "searchApi";
private final IndexNameProvider indexNameProvider;
private final ElasticsearchDocumentConverter documentConverter;

private final RestHighLevelClient client;

public SearchResult<ConsortiumCampus> fetchCampuses(String tenantHeader,
String tenantId,
Integer limit,
Integer offset,
String sortBy,
SortOrder sortOrder) {

var sourceBuilder = getSearchSourceBuilder(tenantId, limit, offset, sortBy, sortOrder);
var response = search(sourceBuilder, tenantHeader);
return documentConverter.convertToSearchResult(response, ConsortiumCampus.class);
}

@NotNull
private static SearchSourceBuilder getSearchSourceBuilder(String tenantId,
Integer limit,
Integer offset,
String sortBy,
SortOrder sortOrder) {
var sourceBuilder = new SearchSourceBuilder();
Optional.ofNullable(tenantId)
.ifPresent(id -> sourceBuilder
.query(QueryBuilders
.termQuery(TENANT_ID_FIELD_NAME, id)));

return sourceBuilder
.from(offset)
.sort(SortBuilders
.fieldSort(sortBy)
.order(sortOrder == SortOrder.DESC ? DESC : ASC))
.size(limit);
}

private SearchResponse search(SearchSourceBuilder sourceBuilder, String tenantHeader) {
var index = indexNameProvider.getIndexName(CAMPUS_INDEX, tenantHeader);
var searchRequest = new SearchRequest(index);

searchRequest.source(sourceBuilder);

return performExceptionalOperation(() -> client.search(searchRequest,
RequestOptions.DEFAULT), index, OPERATION_TYPE);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ protected OutputStreamWriter createOutputStreamWriter(OutputStream outputStream)
private void streamResourceIds(CqlResourceIdsRequest request, Consumer<List<String>> idsConsumer) {
log.info("streamResourceIds:: by [query: {}, resource: {}]", request.getQuery(), request.getResource());

var searchSource = queryConverter.convertForConsortia(request.getQuery(), request.getResource())
var searchSource = queryConverter
.convertForConsortia(request.getQuery(), request.getResource(), request.getTenantId())
.size(streamIdsProperties.getScrollQuerySize())
.fetchSource(new String[] {request.getSourceFieldPath()}, null)
.sort(fieldSort("_doc"));
Expand Down
Loading

0 comments on commit b83a267

Please sign in to comment.