diff --git a/build.gradle b/build.gradle index 18af9b91..ec254337 100644 --- a/build.gradle +++ b/build.gradle @@ -51,6 +51,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.boot:spring-boot-starter-security' + implementation group: 'org.springframework', name: 'spring-web', version: '5.3.32' + implementation 'org.springframework.data:spring-data-elasticsearch' implementation 'org.apache.jena:apache-jena-libs:4.9.0' implementation 'org.apache.jena:jena-querybuilder:4.9.0' diff --git a/src/integration/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerMvcTest.java b/src/integration/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerMvcTest.java index f209ada9..2e160320 100644 --- a/src/integration/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerMvcTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerMvcTest.java @@ -1,11 +1,13 @@ package it.gov.innovazione.ndc.controller; import it.gov.innovazione.ndc.controller.exception.SemanticAssetNotFoundException; +import it.gov.innovazione.ndc.gen.dto.SearchResultItem; import it.gov.innovazione.ndc.gen.dto.SemanticAssetDetails; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.model.Builders; import it.gov.innovazione.ndc.service.SemanticAssetSearchService; -import it.gov.innovazione.ndc.gen.dto.SearchResultItem; import org.junit.jupiter.api.Test; +import org.mockito.Mock; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.boot.test.mock.mockito.MockBean; @@ -33,6 +35,8 @@ public class SemanticAssetsControllerMvcTest { private MockMvc mockMvc; @MockBean private SemanticAssetSearchService searchService; + @MockBean + private RepositoryService repositoryService; @Test void shouldFindByIri() throws Exception { @@ -72,7 +76,7 @@ void shouldReturnMatchingAssetsUsingDefaultPageParams() throws Exception { dto.setDescription("some-description"); dto.setModifiedOn(LocalDate.parse("2020-01-01")); - when(searchService.search(any(), any(), any(), any()) + when(searchService.search(any(), any(), any(), any(), any()) ).thenReturn(Builders.searchResult() .limit(10) .offset(0) @@ -103,13 +107,14 @@ void shouldReturnMatchingAssetsUsingDefaultPageParams() throws Exception { verify(searchService).search("searchText", Set.of("CONTROLLED_VOCABULARY", "ONTOLOGY"), Set.of("http://publications.europa.eu/resource/authority/data-theme/AGRI", "http://publications.europa.eu/resource/authority/data-theme/EDUC"), + null, OffsetBasedPageRequest.of(0, 10)); } @Test void shouldReturnMatchingAssetsUsingProvidedPageParams() throws Exception { SearchResultItem dto = new SearchResultItem(); - when(searchService.search(any(), any(), any(), any()) + when(searchService.search(any(), any(), any(), any(), any()) ).thenReturn(Builders.searchResult() .limit(20) .offset(100) @@ -123,7 +128,7 @@ void shouldReturnMatchingAssetsUsingProvidedPageParams() throws Exception { .accept(MediaType.APPLICATION_JSON) ); - verify(searchService).search("", Set.of(), Set.of(), OffsetBasedPageRequest.of(100, 20)); + verify(searchService).search("", Set.of(), Set.of(), null, OffsetBasedPageRequest.of(100, 20)); apiResult .andDo(print()) @@ -140,7 +145,7 @@ void shouldSearchWithDefaultWhenNoParamsProvided() throws Exception { .andDo(print()) .andExpect(status().isOk()); - verify(searchService).search("", Set.of(), Set.of(), OffsetBasedPageRequest.of(0, 10)); + verify(searchService).search("", Set.of(), Set.of(), null, OffsetBasedPageRequest.of(0, 10)); } @Test diff --git a/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java b/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java index cfc3ab5d..0fd429e8 100644 --- a/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java +++ b/src/integration/java/it/gov/innovazione/ndc/integration/BaseIntegrationTest.java @@ -3,6 +3,7 @@ import it.gov.innovazione.ndc.harvester.AgencyRepositoryService; import it.gov.innovazione.ndc.harvester.HarvesterService; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.repository.TripleStoreProperties; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -23,6 +24,7 @@ import java.util.Set; import static it.gov.innovazione.ndc.harvester.service.RepositoryUtils.asRepo; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; @@ -49,6 +51,9 @@ public class BaseIntegrationTest { @SpyBean AgencyRepositoryService agencyRepositoryService; + @SpyBean + RepositoryService repositoryService; + @Autowired TripleStoreProperties virtuosoProps; @@ -79,6 +84,7 @@ private void dataIsHarvested() throws IOException { Path cloneDir = Path.of(dir); doReturn(cloneDir).when(agencyRepositoryService).cloneRepo(REPO_URL, null); doNothing().when(agencyRepositoryService).removeClonedRepo(cloneDir); + doNothing().when(repositoryService).storeRightsHolders(any(), any()); harvesterService.harvest(asRepo(REPO_URL)); diff --git a/src/main/java/it/gov/innovazione/ndc/config/HarvestExecutionContext.java b/src/main/java/it/gov/innovazione/ndc/config/HarvestExecutionContext.java index d61e83bb..7a88876b 100644 --- a/src/main/java/it/gov/innovazione/ndc/config/HarvestExecutionContext.java +++ b/src/main/java/it/gov/innovazione/ndc/config/HarvestExecutionContext.java @@ -1,12 +1,17 @@ package it.gov.innovazione.ndc.config; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.AccessLevel; import lombok.Builder; import lombok.Data; import lombok.RequiredArgsConstructor; +import lombok.Singular; import lombok.With; +import java.util.ArrayList; +import java.util.List; + @With @Data @Builder @@ -18,4 +23,10 @@ public class HarvestExecutionContext { private final String runId; private final String currentUserId; private final String rootPath; + @Singular + private final List rightsHolders = new ArrayList<>(); + + public void addRightsHolder(RightsHolder agencyId) { + rightsHolders.add(agencyId); + } } diff --git a/src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java b/src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java index d11d9418..04512ce0 100644 --- a/src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java +++ b/src/main/java/it/gov/innovazione/ndc/controller/SemanticAssetsController.java @@ -1,18 +1,28 @@ package it.gov.innovazione.ndc.controller; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiResponse; +import io.swagger.annotations.ApiResponses; import it.gov.innovazione.ndc.gen.api.SemanticAssetsApi; import it.gov.innovazione.ndc.gen.dto.AssetType; +import it.gov.innovazione.ndc.gen.dto.Direction; import it.gov.innovazione.ndc.gen.dto.SearchResult; import it.gov.innovazione.ndc.gen.dto.SemanticAssetDetails; +import it.gov.innovazione.ndc.gen.dto.SortBy; import it.gov.innovazione.ndc.gen.dto.Theme; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.service.SemanticAssetSearchService; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Pageable; import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import java.net.URI; import java.util.Collections; +import java.util.List; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -21,15 +31,45 @@ @RestController public class SemanticAssetsController implements SemanticAssetsApi { private final SemanticAssetSearchService searchService; + private final RepositoryService repositoryService; + + /** + * GET /semantic-assets/rights-holders + * Retrieves the rights holders of the semantic assets. + * + * @return OK (status code 200) + */ + @ApiOperation(value = "", nickname = "getRightsHolders", notes = "Retrieves the rights holders", + response = SemanticAssetDetails.class, tags = {"semantic-assets"}) + @ApiResponses(value = {@ApiResponse(code = 200, message = "OK", response = SemanticAssetDetails.class)}) + @RequestMapping( + method = RequestMethod.GET, + value = "/semantic-assets/rights-holders", + produces = {"application/json"} + ) + List getRightsHolders() { + return repositoryService.getRightsHolders(); + + } @Override - public ResponseEntity search(String q, Integer offset, Integer limit, Set type, Set theme) { + public ResponseEntity search( + String q, + Integer offset, + Integer limit, + SortBy sortBy, + Direction direction, + Set type, + Set theme, + Set rightsHolder) { + Pageable pageable = OffsetBasedPageRequest.of(offset, limit); return AppJsonResponse.ok( searchService.search(q, toEnumStrings(type, AssetType::getValue), toEnumStrings(theme, Theme::getValue), + rightsHolder, pageable ) ); diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java b/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java index 8c6d6252..f54e277d 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/HarvesterService.java @@ -2,6 +2,8 @@ import it.gov.innovazione.ndc.config.HarvestExecutionContext; import it.gov.innovazione.ndc.config.HarvestExecutionContextUtils; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.model.harvester.Repository; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; import it.gov.innovazione.ndc.repository.TripleStoreRepository; @@ -11,8 +13,15 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; @Slf4j @Component @@ -22,6 +31,7 @@ public class HarvesterService { private final List semanticAssetHarvesters; private final TripleStoreRepository tripleStoreRepository; private final SemanticAssetMetadataRepository semanticAssetMetadataRepository; + private final RepositoryService repositoryService; public void harvest(Repository repository) throws IOException { harvest(repository, null); @@ -76,10 +86,22 @@ private void harvestClonedRepo(Repository repository, Path path) { clearRepo(repository.getUrl()); harvestSemanticAssets(repository, path); + storeRightsHolders(repository); log.info("Repo {} processed", repository); } + private void storeRightsHolders(Repository repository) { + Map> rightsHolders = Optional.ofNullable(HarvestExecutionContextUtils.getContext()) + .map(HarvestExecutionContext::getRightsHolders) + .orElse(Collections.emptyList()).stream() + .collect(groupingBy(RightsHolder::getIdentifier, toList())) + .entrySet().stream() + .collect(toMap(Map.Entry::getKey, e -> e.getValue().get(0).getName())); + + repositoryService.storeRightsHolders(repository, rightsHolders); + } + private void clearRepo(String repoUrl) { cleanUpWithHarvesters(repoUrl); cleanUpTripleStore(repoUrl); diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java index a601d557..0f0fd815 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/BaseSemanticAssetModel.java @@ -6,6 +6,7 @@ import it.gov.innovazione.ndc.harvester.model.extractors.NodeExtractor; import it.gov.innovazione.ndc.harvester.model.index.Distribution; import it.gov.innovazione.ndc.harvester.model.index.NodeSummary; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.model.profiles.Admsapit; import lombok.Getter; @@ -36,6 +37,7 @@ import static it.gov.innovazione.ndc.harvester.model.extractors.NodeExtractor.requireNodes; import static it.gov.innovazione.ndc.harvester.model.extractors.NodeSummaryExtractor.extractRequiredNodeSummary; import static it.gov.innovazione.ndc.harvester.model.extractors.NodeSummaryExtractor.maybeNodeSummaries; +import static it.gov.innovazione.ndc.harvester.model.extractors.RightsHolderExtractor.getAgencyId; import static java.lang.String.format; import static java.util.Collections.emptyList; import static org.apache.jena.rdf.model.ResourceFactory.createResource; @@ -200,6 +202,7 @@ protected static List extractDistributionsFilteredByFormat( public SemanticAssetMetadata extractMetadata() { Resource mainResource = getMainResource(); + RightsHolder agencyId = getAgencyId(mainResource, validationContext); return SemanticAssetMetadata.builder() .iri(mainResource.getURI()) .repoUrl(repoUrl) @@ -221,6 +224,7 @@ public SemanticAssetMetadata extractMetadata() { .conformsTo(maybeNodeSummaries(mainResource, conformsTo, FOAF.name)) .distributions(getDistributions()) .status(LiteralExtractor.extractAll(mainResource, Admsapit.status)) + .agencyId(agencyId.getIdentifier()) .build(); } @@ -243,6 +247,7 @@ public SemanticAssetModelValidationContext validateMetadata() { .add(v -> extractOptional(getMainResource(), temporal, v.withWarningValidationType().withFieldName(SemanticAssetMetadata.Fields.temporal))) .add(v -> maybeNodeSummaries(getMainResource(), conformsTo, FOAF.name, v.withWarningValidationType().withFieldName(SemanticAssetMetadata.Fields.conformsTo))) .add(v -> getDistributions(v.withFieldName(SemanticAssetMetadata.Fields.distributions))) + .add(v -> getAgencyId(getMainResource(), v.withFieldName(SemanticAssetMetadata.Fields.agencyId))) .build() .stream() .map(consumer -> returningValidationContext(this.validationContext, consumer)) diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java index ac774baf..6e4da965 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModel.java @@ -2,7 +2,9 @@ import com.github.jsonldjava.shaded.com.google.common.collect.ImmutableList; import it.gov.innovazione.ndc.harvester.model.exception.InvalidModelException; +import it.gov.innovazione.ndc.harvester.model.extractors.RightsHolderExtractor; import it.gov.innovazione.ndc.harvester.model.index.Distribution; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.model.profiles.NDC; import lombok.extern.slf4j.Slf4j; @@ -10,7 +12,6 @@ import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.Statement; import org.apache.jena.rdf.model.StmtIterator; -import org.apache.jena.vocabulary.DCTerms; import org.apache.jena.vocabulary.RDF; import java.util.List; @@ -90,29 +91,8 @@ private static void validateKeyConcept(String keyConcept, Resource mainResource, } } - public String getAgencyId() { - return getAgencyId(getMainResource(), NO_VALIDATION); - } - - public static String getAgencyId(Resource mainResource, SemanticAssetModelValidationContext validationContext) { - Statement rightsHolder; - try { - rightsHolder = mainResource.getRequiredProperty(DCTerms.rightsHolder); - } catch (Exception e) { - InvalidModelException invalidModelException = new InvalidModelException(format("Cannot find required rightsHolder property (%s)", DCTerms.rightsHolder)); - validationContext.addValidationException(invalidModelException); - throw invalidModelException; - } - Statement idProperty; - try { - idProperty = rightsHolder.getProperty(DCTerms.identifier); - } catch (Exception e) { - String rightsHolderIri = rightsHolder.getObject().toString(); - InvalidModelException invalidModelException = new InvalidModelException(format("Cannot find required id (%s) for rightsHolder '%s'", DCTerms.identifier, rightsHolderIri)); - validationContext.addValidationException(invalidModelException); - throw invalidModelException; - } - return idProperty.getString(); + public RightsHolder getAgencyId() { + return RightsHolderExtractor.getAgencyId(getMainResource(), NO_VALIDATION); } public void addNdcDataServiceProperties(String baseUrl) { @@ -125,7 +105,7 @@ public void addNdcDataServiceProperties(String baseUrl) { } private String buildDataServiceIndividualUri() { - return format("https://w3id.org/italia/data/data-service/%s-%s", getAgencyId(), getKeyConcept()); + return format("https://w3id.org/italia/data/data-service/%s-%s", getAgencyId().getIdentifier(), getKeyConcept()); } public String getEndpointUrl() { @@ -133,7 +113,7 @@ public String getEndpointUrl() { } private String buildEndpointUrl(String baseUrl) { - return format(NDC_ENDPOINT_URL_TEMPLATE, baseUrl, getAgencyId(), getKeyConcept()); + return format(NDC_ENDPOINT_URL_TEMPLATE, baseUrl, getAgencyId().getIdentifier(), getKeyConcept()); } @Override @@ -147,7 +127,6 @@ public SemanticAssetMetadata extractMetadata() { .type(CONTROLLED_VOCABULARY) .distributions(getDistributions()) .keyConcept(getKeyConcept()) - .agencyId(getAgencyId()) .endpointUrl(getEndpointUrl()) .build(); } @@ -159,7 +138,7 @@ public SemanticAssetModelValidationContext validateMetadata() { SemanticAssetModelValidationContext context = new ImmutableList.Builder>() .add(v -> getDistributions(v.withFieldName(SemanticAssetMetadata.Fields.distributions))) .add(v -> getKeyConcept(getMainResource(), v.withWarningValidationType().withFieldName(SemanticAssetMetadata.Fields.keyConcept))) - .add(v -> getAgencyId(getMainResource(), v.withWarningValidationType().withFieldName(SemanticAssetMetadata.Fields.agencyId))) + .add(v -> RightsHolderExtractor.getAgencyId(getMainResource(), v.withWarningValidationType().withFieldName(SemanticAssetMetadata.Fields.agencyId))) .build() .stream() .map(consumer -> returningValidationContext(this.validationContext, consumer)) diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java index 7ff3d357..4c5fd7d9 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/SchemaModel.java @@ -4,8 +4,10 @@ import it.gov.innovazione.ndc.harvester.model.extractors.LiteralExtractor; import it.gov.innovazione.ndc.harvester.model.extractors.NodeExtractor; import it.gov.innovazione.ndc.harvester.model.extractors.NodeSummaryExtractor; +import it.gov.innovazione.ndc.harvester.model.extractors.RightsHolderExtractor; import it.gov.innovazione.ndc.harvester.model.index.Distribution; import it.gov.innovazione.ndc.harvester.model.index.NodeSummary; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.model.profiles.Admsapit; import org.apache.jena.rdf.model.Model; @@ -57,6 +59,7 @@ protected String getMainResourceTypeIri() { @Override public SemanticAssetMetadata extractMetadata() { Resource mainResource = getMainResource(); + RightsHolder rightsHolderObj = RightsHolderExtractor.getAgencyId(mainResource, validationContext); return SemanticAssetMetadata.builder() .iri(mainResource.getURI()) .repoUrl(repoUrl) @@ -73,6 +76,7 @@ public SemanticAssetMetadata extractMetadata() { .conformsTo(NodeSummaryExtractor.maybeNodeSummaries(mainResource, conformsTo, FOAF.name, validationContext)) .keyClasses(getKeyClasses()) .status(LiteralExtractor.extractAll(mainResource, Admsapit.status)) + .agencyId(rightsHolderObj.getIdentifier()) .build(); } diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/LiteralExtractor.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/LiteralExtractor.java index a3356cb5..6a134c5e 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/LiteralExtractor.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/LiteralExtractor.java @@ -4,11 +4,13 @@ import it.gov.innovazione.ndc.harvester.model.exception.InvalidModelException; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.tuple.Pair; import org.apache.jena.rdf.model.Property; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.Statement; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.Predicate; @@ -16,6 +18,7 @@ import static it.gov.innovazione.ndc.harvester.model.SemanticAssetModelValidationContext.NO_VALIDATION; import static java.lang.String.format; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; @NoArgsConstructor(access = AccessLevel.PRIVATE) public class LiteralExtractor { @@ -54,6 +57,13 @@ public static String extract(Resource resource, Property property, SemanticAsset return null; } + public static Map extractAllLanguages(Resource resource, Property property) { + return resource.listProperties(property).toList().stream() + .filter(s -> s.getObject().isLiteral()) + .map(s -> Pair.of(s.getLanguage(), s.getString())) + .collect(toMap(Pair::getLeft, Pair::getRight)); + } + public static List extractAll(Resource resource, Property property) { return resource.listProperties(property).toList().stream() .map(Statement::getString) diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/RightsHolderExtractor.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/RightsHolderExtractor.java new file mode 100644 index 00000000..9595e88f --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/extractors/RightsHolderExtractor.java @@ -0,0 +1,44 @@ +package it.gov.innovazione.ndc.harvester.model.extractors; + +import it.gov.innovazione.ndc.harvester.model.SemanticAssetModelValidationContext; +import it.gov.innovazione.ndc.harvester.model.exception.InvalidModelException; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; +import org.apache.jena.rdf.model.Resource; +import org.apache.jena.rdf.model.Statement; +import org.apache.jena.sparql.vocabulary.FOAF; +import org.apache.jena.vocabulary.DCTerms; + +import java.util.Map; + +import static it.gov.innovazione.ndc.harvester.model.extractors.LiteralExtractor.extractAllLanguages; +import static java.lang.String.format; + +public class RightsHolderExtractor { + + public static RightsHolder getAgencyId(Resource mainResource, SemanticAssetModelValidationContext validationContext) { + Statement rightsHolder; + try { + rightsHolder = mainResource.getRequiredProperty(DCTerms.rightsHolder); + } catch (Exception e) { + InvalidModelException invalidModelException = new InvalidModelException(format("Cannot find required rightsHolder property (%s)", DCTerms.rightsHolder)); + validationContext.addValidationException(invalidModelException); + throw invalidModelException; + } + Statement idProperty; + try { + idProperty = rightsHolder.getProperty(DCTerms.identifier); + } catch (Exception e) { + String rightsHolderIri = rightsHolder.getObject().toString(); + InvalidModelException invalidModelException = new InvalidModelException(format("Cannot find required id (%s) for rightsHolder '%s'", DCTerms.identifier, rightsHolderIri)); + validationContext.addValidationException(invalidModelException); + throw invalidModelException; + } + Map names = extractAllLanguages(rightsHolder.getResource(), FOAF.name); + + return RightsHolder.builder() + .identifier(idProperty.getString()) + .name(names) + .build(); + } + +} diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/index/RightsHolder.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/RightsHolder.java new file mode 100644 index 00000000..f29a04d7 --- /dev/null +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/RightsHolder.java @@ -0,0 +1,13 @@ +package it.gov.innovazione.ndc.harvester.model.index; + +import lombok.Builder; +import lombok.Data; + +import java.util.Map; + +@Data +@Builder +public class RightsHolder { + private String identifier; + private Map name; +} diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java index 5a8987aa..fb163909 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/model/index/SemanticAssetMetadata.java @@ -18,7 +18,7 @@ import static org.springframework.data.elasticsearch.annotations.FieldType.Keyword; import static org.springframework.data.elasticsearch.annotations.FieldType.Text; -@Document(indexName = "semantic-asset-metadata") +@Document(indexName = "semantic-asset-metadata-2") @Setting(settingPath = "elasticsearch-settings.json") @Data @Builder(toBuilder = true) @@ -74,7 +74,7 @@ public class SemanticAssetMetadata { // Controlled Vocabulary Specific @Field(index = false, type = Keyword) private String keyConcept; - @Field(index = false, type = Keyword) + @Field(type = Keyword) private String agencyId; @Field(index = false, type = Keyword) private String endpointUrl; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/BaseSemanticAssetPathProcessor.java b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/BaseSemanticAssetPathProcessor.java index 942f949c..c6c788d5 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/BaseSemanticAssetPathProcessor.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/BaseSemanticAssetPathProcessor.java @@ -1,8 +1,11 @@ package it.gov.innovazione.ndc.harvester.pathprocessors; +import it.gov.innovazione.ndc.config.HarvestExecutionContext; +import it.gov.innovazione.ndc.config.HarvestExecutionContextUtils; import it.gov.innovazione.ndc.harvester.exception.SinglePathProcessingException; import it.gov.innovazione.ndc.harvester.model.SemanticAssetModel; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; +import it.gov.innovazione.ndc.harvester.model.extractors.RightsHolderExtractor; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; import it.gov.innovazione.ndc.repository.TripleStoreRepository; @@ -10,6 +13,8 @@ import lombok.extern.slf4j.Slf4j; import org.apache.jena.rdf.model.Resource; +import static it.gov.innovazione.ndc.harvester.model.SemanticAssetModelValidationContext.NO_VALIDATION; + @RequiredArgsConstructor @Slf4j public abstract class BaseSemanticAssetPathProcessor

implements SemanticAssetPathProcessor

{ @@ -41,6 +46,16 @@ protected void processWithModel(String repoUrl, P path, M model) { enrichModelBeforePersisting(model, path); indexMetadataForSearch(model); persistModelToTripleStore(repoUrl, path, model); + collectRightsHolderInContext(repoUrl, model); + } + + private void collectRightsHolderInContext(String repoUrl, M model) { + try { + HarvestExecutionContext context = HarvestExecutionContextUtils.getContext(); + context.addRightsHolder(RightsHolderExtractor.getAgencyId(model.getMainResource(), NO_VALIDATION)); + } catch (Exception e) { + log.error("Error adding rights holder to repo " + repoUrl, e); + } } protected void enrichModelBeforePersisting(M model, P path) { diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java index d7d0c706..2dc18b73 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessor.java @@ -1,6 +1,5 @@ package it.gov.innovazione.ndc.harvester.pathprocessors; -import it.gov.innovazione.ndc.repository.TripleStoreRepository; import it.gov.innovazione.ndc.harvester.csv.CsvParser; import it.gov.innovazione.ndc.harvester.csv.CsvParser.CsvData; import it.gov.innovazione.ndc.harvester.model.ControlledVocabularyModel; @@ -8,6 +7,7 @@ import it.gov.innovazione.ndc.harvester.model.SemanticAssetModelFactory; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; +import it.gov.innovazione.ndc.repository.TripleStoreRepository; import it.gov.innovazione.ndc.service.VocabularyDataService; import it.gov.innovazione.ndc.service.VocabularyIdentifier; import lombok.extern.slf4j.Slf4j; @@ -41,7 +41,7 @@ protected void processWithModel(String repoUrl, CvPath path, ControlledVocabular path.getCsvPath().ifPresent(p -> { String keyConcept = model.getKeyConcept(); - String agencyId = model.getAgencyId(); + String agencyId = model.getAgencyId().getIdentifier(); VocabularyIdentifier vocabularyIdentifier = new VocabularyIdentifier(agencyId, keyConcept); parseAndIndexCsv(vocabularyIdentifier, p); diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/OntologyPathProcessor.java b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/OntologyPathProcessor.java index d7f24a60..fbe3fb70 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/OntologyPathProcessor.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/pathprocessors/OntologyPathProcessor.java @@ -1,10 +1,10 @@ package it.gov.innovazione.ndc.harvester.pathprocessors; +import it.gov.innovazione.ndc.harvester.model.OntologyModel; import it.gov.innovazione.ndc.harvester.model.SemanticAssetModelFactory; import it.gov.innovazione.ndc.harvester.model.SemanticAssetPath; -import it.gov.innovazione.ndc.repository.TripleStoreRepository; -import it.gov.innovazione.ndc.harvester.model.OntologyModel; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; +import it.gov.innovazione.ndc.repository.TripleStoreRepository; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; diff --git a/src/main/java/it/gov/innovazione/ndc/harvester/service/RepositoryService.java b/src/main/java/it/gov/innovazione/ndc/harvester/service/RepositoryService.java index 59daf66f..97aff609 100644 --- a/src/main/java/it/gov/innovazione/ndc/harvester/service/RepositoryService.java +++ b/src/main/java/it/gov/innovazione/ndc/harvester/service/RepositoryService.java @@ -1,6 +1,8 @@ package it.gov.innovazione.ndc.harvester.service; +import com.fasterxml.jackson.databind.ObjectMapper; import it.gov.innovazione.ndc.controller.RepositoryController; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; import it.gov.innovazione.ndc.model.harvester.Repository; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -11,11 +13,16 @@ import java.security.Principal; import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.mapping; +import static java.util.stream.Collectors.toList; import static org.springframework.util.StringUtils.startsWithIgnoreCase; @Service @@ -35,10 +42,12 @@ public class RepositoryService { + "CREATED_BY, " + "UPDATED, " + "UPDATED_BY, " - + "MAX_FILE_SIZE_BYTES " + + "MAX_FILE_SIZE_BYTES, " + + "RIGHTS_HOLDER " + "FROM REPOSITORY"; private final JdbcTemplate jdbcTemplate; + private final ObjectMapper objectMapper; @Value("#{'${harvester.repositories}'}") private final String repositories; @@ -59,13 +68,14 @@ public List getAllRepos() { .updatedAt(rs.getTimestamp("UPDATED").toInstant()) .updatedBy(rs.getString("UPDATED_BY")) .maxFileSizeBytes(rs.getLong("MAX_FILE_SIZE_BYTES")) + .rightsHolders(readSafely(rs.getString("RIGHTS_HOLDER"))) .build()); if (!allRepos.isEmpty()) { allRepos.forEach(repo -> log.info("Repository: " + repo.toString())); return allRepos.stream() .filter(Repository::getActive) - .collect(Collectors.toList()); + .collect(toList()); } log.warn("No repositories found in the database. Using the default repositories from configuration"); @@ -76,13 +86,22 @@ public List getAllRepos() { .orElse(emptyList()) .stream() .map(RepositoryUtils::asRepo) - .collect(Collectors.toList()); + .collect(toList()); saveDefaultRepositories(defaultRepositories); return defaultRepositories; } + private Map> readSafely(String rightsHolders) { + try { + return objectMapper.readValue(rightsHolders, Map.class); + } catch (Exception e) { + log.error("Error reading rights holders", e); + return null; + } + } + private void saveDefaultRepositories(List defaultRepositories) { defaultRepositories.forEach(this::save); } @@ -194,4 +213,33 @@ public int delete(String id, Principal principal) { principal.getName(), id); } + + @SneakyThrows + public void storeRightsHolders(Repository repository, Map> rightsHolders) { + log.info("Storing rights holders for repository {}", repository); + String query = "UPDATE REPOSITORY SET " + + "RIGHTS_HOLDER = ? " + + "WHERE ID = ?"; + jdbcTemplate.update(query, + objectMapper.writeValueAsString(rightsHolders), + repository.getId()); + } + + public List getRightsHolders() { + return getAllRepos().stream() + .map(Repository::getRightsHolders) + .map(Map::entrySet) + .flatMap(Collection::stream) + .collect(groupingBy( + Map.Entry::getKey, + mapping( + Map.Entry::getValue, + toList()))) + .entrySet().stream() + .map(entry -> RightsHolder.builder() + .identifier(entry.getKey()) + .name(entry.getValue().get(0)) + .build()) + .collect(Collectors.toList()); + } } diff --git a/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java b/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java index 2041756b..c8b2171e 100644 --- a/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java +++ b/src/main/java/it/gov/innovazione/ndc/model/harvester/Repository.java @@ -6,6 +6,7 @@ import lombok.With; import java.time.Instant; +import java.util.Map; @Data @Builder(toBuilder = true) @@ -23,4 +24,5 @@ public class Repository { private Instant updatedAt; private String updatedBy; private Long maxFileSizeBytes; + private Map> rightsHolders; } diff --git a/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java b/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java index e826970f..4a96a123 100644 --- a/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java +++ b/src/main/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepository.java @@ -1,18 +1,6 @@ package it.gov.innovazione.ndc.repository; -import static it.gov.innovazione.ndc.harvester.SemanticAssetType.CONTROLLED_VOCABULARY; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.springframework.data.elasticsearch.core.SearchHitSupport.searchPageFor; - -import it.gov.innovazione.ndc.harvester.SemanticAssetType; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; - -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.elasticsearch.common.unit.Fuzziness; @@ -20,7 +8,6 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.TermsQueryBuilder; import org.springframework.data.domain.Pageable; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; @@ -32,6 +19,17 @@ import org.springframework.stereotype.Repository; import org.springframework.util.ObjectUtils; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static it.gov.innovazione.ndc.harvester.SemanticAssetType.CONTROLLED_VOCABULARY; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.springframework.data.elasticsearch.core.SearchHitSupport.searchPageFor; + @Repository @RequiredArgsConstructor @Slf4j @@ -39,11 +37,12 @@ public class SemanticAssetMetadataRepository { private final ElasticsearchOperations esOps; public SearchPage search(String queryPattern, Set types, - Set themes, Pageable pageable) { + Set themes, Set rightsHolder, + Pageable pageable) { BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(matchQuery("searchableText", queryPattern)); - addFilters(types, themes, boolQuery); + addFilters(types, themes, rightsHolder, boolQuery); NativeSearchQuery query = new NativeSearchQueryBuilder() .withQuery(boolQuery) @@ -65,13 +64,16 @@ public void save(SemanticAssetMetadata metadata) { esOps.save(metadata); } - private void addFilters(Set types, Set themes, BoolQueryBuilder finalQuery) { + private void addFilters(Set types, Set themes, Set rightsHolder, BoolQueryBuilder finalQuery) { if (!types.isEmpty()) { finalQuery.filter(new TermsQueryBuilder("type", types)); } if (!themes.isEmpty()) { finalQuery.filter(new TermsQueryBuilder("themes", themes)); } + if (Objects.nonNull(rightsHolder) && !rightsHolder.isEmpty()) { + finalQuery.filter(new TermsQueryBuilder("agencyId", rightsHolder)); + } } private QueryBuilder matchQuery(String field, String value) { diff --git a/src/main/java/it/gov/innovazione/ndc/service/SemanticAssetSearchService.java b/src/main/java/it/gov/innovazione/ndc/service/SemanticAssetSearchService.java index 9e1b26aa..00733b31 100644 --- a/src/main/java/it/gov/innovazione/ndc/service/SemanticAssetSearchService.java +++ b/src/main/java/it/gov/innovazione/ndc/service/SemanticAssetSearchService.java @@ -1,13 +1,12 @@ package it.gov.innovazione.ndc.service; -import it.gov.innovazione.ndc.controller.OffsetBasedPageRequest; -import it.gov.innovazione.ndc.gen.dto.AssetType; -import it.gov.innovazione.ndc.gen.dto.VocabulariesResult; -import it.gov.innovazione.ndc.model.SemanticAssetsMetadataMapper; import it.gov.innovazione.ndc.controller.exception.SemanticAssetNotFoundException; +import it.gov.innovazione.ndc.gen.dto.AssetType; import it.gov.innovazione.ndc.gen.dto.SearchResult; import it.gov.innovazione.ndc.gen.dto.SemanticAssetDetails; +import it.gov.innovazione.ndc.gen.dto.VocabulariesResult; import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; +import it.gov.innovazione.ndc.model.SemanticAssetsMetadataMapper; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; import lombok.RequiredArgsConstructor; import org.springframework.data.domain.Pageable; @@ -24,10 +23,10 @@ public class SemanticAssetSearchService { private final SemanticAssetsMetadataMapper mapper; public SearchResult search(String queryPattern, Set types, - Set themes, Pageable pageable) { + Set themes, Set rightsHolder, Pageable pageable) { SearchPage searchResults = - metadataRepository.search(queryPattern, types, themes, pageable); + metadataRepository.search(queryPattern, types, themes, rightsHolder, pageable); return mapper.searchResultToDto(searchResults); } @@ -41,7 +40,7 @@ public SemanticAssetDetails findByIri(String iri) { public VocabulariesResult getVocabularies(Pageable pageable) { SearchPage results = metadataRepository.search("", Set.of(AssetType.CONTROLLED_VOCABULARY.getValue()), - Collections.emptySet(), pageable); + Collections.emptySet(), Collections.emptySet(), pageable); return mapper.vocabResultToDto(results); } } diff --git a/src/main/resources/db/migration/V5__rights_holder.sql b/src/main/resources/db/migration/V5__rights_holder.sql new file mode 100644 index 00000000..015fa8a4 --- /dev/null +++ b/src/main/resources/db/migration/V5__rights_holder.sql @@ -0,0 +1,2 @@ +alter table REPOSITORY + add RIGHTS_HOLDER varchar(8192) default '{}' not null; diff --git a/src/main/resources/public/openapi.yaml b/src/main/resources/public/openapi.yaml index e3437509..b0634515 100644 --- a/src/main/resources/public/openapi.yaml +++ b/src/main/resources/public/openapi.yaml @@ -157,8 +157,10 @@ paths: type: string default: '' maxLength: 4096 - - $ref: "#/components/parameters/offset" + - $ref: "#/components/parameters/offset" - $ref: "#/components/parameters/limit" + - $ref: "#/components/parameters/sortBy" + - $ref: "#/components/parameters/direction" - name: type in: query required: false @@ -181,6 +183,17 @@ paths: items: $ref: "#/components/schemas/Theme" default: [ ] + - name: rightsHolder + in: query + required: false + schema: + uniqueItems: true + type: array + minItems: 0 + maxItems: *max-array-size-default + items: + type: string + default: [ ] responses: '200': description: OK @@ -229,6 +242,18 @@ components: required: false schema: $ref: "#/components/schemas/Offset" + sortBy: + name: sortBy + in: query + required: false + schema: + $ref: "#/components/schemas/SortBy" + direction: + name: direction + in: query + required: false + schema: + $ref: "#/components/schemas/Direction" agency_id: name: agency_id in: path @@ -257,6 +282,18 @@ components: maximum: 200 minimum: 1 default: 10 + SortBy: + type: string + enum: + - TITLE + - LAST_MODIFIED + default: "TITLE" + Direction: + type: string + enum: + - ASC + - DESC + default: ASC VocabularyData: type: object additionalProperties: false @@ -360,6 +397,10 @@ components: $ref: "#/components/schemas/Limit" offset: $ref: "#/components/schemas/Offset" + sortBy: + $ref: "#/components/schemas/SortBy" + direction: + $ref: "#/components/schemas/Direction" data: type: array minItems: 0 diff --git a/src/test/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerTest.java b/src/test/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerTest.java index 7bb6f59c..489b3e43 100644 --- a/src/test/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerTest.java +++ b/src/test/java/it/gov/innovazione/ndc/controller/SemanticAssetsControllerTest.java @@ -3,6 +3,7 @@ import it.gov.innovazione.ndc.gen.dto.SearchResult; import it.gov.innovazione.ndc.gen.dto.SemanticAssetDetails; import it.gov.innovazione.ndc.gen.dto.Theme; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.model.Builders; import it.gov.innovazione.ndc.service.SemanticAssetSearchService; import org.junit.jupiter.api.Test; @@ -25,28 +26,33 @@ public class SemanticAssetsControllerTest { @Mock private SemanticAssetSearchService service; + @Mock + private RepositoryService repositoryService; @Test void shouldFetchResultsFromRepositoryForGivenKeyword() { - SemanticAssetsController controller = new SemanticAssetsController(service); + SemanticAssetsController controller = new SemanticAssetsController(service, repositoryService); SearchResult expectedResult = Builders.searchResult().build(); - when(service.search(any(), any(), any(), any())).thenReturn(expectedResult); + when(service.search(any(), any(), any(), any(), any())).thenReturn(expectedResult); SearchResult actualResult = controller.search("searchTerm", 0, 10, + null, null, Set.of(CONTROLLED_VOCABULARY), - Set.of(Theme.EDUC)).getBody(); + Set.of(Theme.EDUC), + Set.of()).getBody(); verify(service).search("searchTerm", Set.of("CONTROLLED_VOCABULARY"), Set.of("http://publications.europa.eu/resource/authority/data-theme/EDUC"), + Set.of(), OffsetBasedPageRequest.of(0, 10)); assertThat(actualResult).isEqualTo(expectedResult); } @Test void shouldGetDetailsOfTheAssetByIri() throws URISyntaxException { - SemanticAssetsController controller = new SemanticAssetsController(service); + SemanticAssetsController controller = new SemanticAssetsController(service, repositoryService); SemanticAssetDetails expected = new SemanticAssetDetails(); when(service.findByIri(any())).thenReturn(expected); diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java index 1dda94c6..7c29b2d5 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/HarvesterServiceTest.java @@ -1,5 +1,6 @@ package it.gov.innovazione.ndc.harvester; +import it.gov.innovazione.ndc.harvester.service.RepositoryService; import it.gov.innovazione.ndc.model.harvester.Repository; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; import it.gov.innovazione.ndc.repository.TripleStoreRepository; @@ -36,6 +37,8 @@ class HarvesterServiceTest { private SemanticAssetMetadataRepository metadataRepository; @Mock private SemanticAssetHarvester harvester; + @Mock + private RepositoryService repositoryService; private HarvesterService harvesterService; @@ -45,7 +48,8 @@ void setUp() { agencyRepoService, List.of(harvester), tripleStoreRepository, - metadataRepository); + metadataRepository, + repositoryService); } @Test diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java index 5b00696f..c4df5790 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/ControlledVocabularyModelTest.java @@ -12,7 +12,6 @@ import org.apache.jena.rdf.model.Statement; import org.apache.jena.vocabulary.RDF; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; @@ -230,7 +229,7 @@ void shouldExtractAgencyId() { ControlledVocabularyModel model = new ControlledVocabularyModel(jenaModel, TTL_FILE, REPO_URL); - assertThat(model.getAgencyId()).isEqualTo("agid"); + assertThat(model.getAgencyId().getIdentifier()).isEqualTo("agid"); } @Test diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java index 1eaf13ef..b37f7acb 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/model/SchemaModelTest.java @@ -9,6 +9,7 @@ import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.Resource; import org.apache.jena.sparql.vocabulary.FOAF; +import org.apache.jena.vocabulary.DCTerms; import org.apache.jena.vocabulary.RDF; import org.apache.jena.vocabulary.VCARD4; import org.junit.jupiter.api.BeforeEach; @@ -56,7 +57,8 @@ void setupMockModel() { jenaModel = createDefaultModel(); Resource agid = jenaModel .createResource(RIGHTS_HOLDER_IRI) - .addProperty(FOAF.name, "agid"); + .addProperty(FOAF.name, "agid") + .addProperty(DCTerms.identifier, "agid"); Resource jsonDistribution = jenaModel.createResource(SCHEMA_IRI + "/dist/json") .addProperty(format, EuropePublicationVocabulary.FILE_TYPE_JSON) .addProperty(accessURL, jenaModel.createResource(SCHEMA_IRI + "/dist/json/index.html")) diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java index b7926c01..93542793 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/ControlledVocabularyPathProcessorTest.java @@ -1,13 +1,14 @@ package it.gov.innovazione.ndc.harvester.pathprocessors; -import it.gov.innovazione.ndc.repository.TripleStoreRepository; import it.gov.innovazione.ndc.harvester.csv.CsvParser; import it.gov.innovazione.ndc.harvester.csv.CsvParser.CsvData; import it.gov.innovazione.ndc.harvester.model.ControlledVocabularyModel; import it.gov.innovazione.ndc.harvester.model.CvPath; -import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.harvester.model.SemanticAssetModelFactory; +import it.gov.innovazione.ndc.harvester.model.index.RightsHolder; +import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; import it.gov.innovazione.ndc.repository.SemanticAssetMetadataRepository; +import it.gov.innovazione.ndc.repository.TripleStoreRepository; import it.gov.innovazione.ndc.service.VocabularyDataService; import it.gov.innovazione.ndc.service.VocabularyIdentifier; import org.apache.jena.rdf.model.Model; @@ -64,7 +65,7 @@ void shouldProcessCsv() { when(semanticAssetModelFactory.createControlledVocabulary(ttlFile, "some-repo")).thenReturn(cvModel); when(cvModel.getRdfModel()).thenReturn(jenaModel); when(cvModel.getKeyConcept()).thenReturn("keyConcept"); - when(cvModel.getAgencyId()).thenReturn("agencyId"); + when(cvModel.getAgencyId()).thenReturn(RightsHolder.builder().identifier("agencyId").build()); CsvData csvData = new CsvData(List.of(Map.of("key", "val")), "key"); when(csvParser.loadCsvDataFromFile(csvFile)).thenReturn(csvData); SemanticAssetMetadata metadata = SemanticAssetMetadata.builder().build(); @@ -150,4 +151,4 @@ private List buildVocabsMetadataWithAgencyAndConcepts(Str SemanticAssetMetadata template = SemanticAssetMetadata.builder().repoUrl(REPO_URL).agencyId(agencyId).build(); return keyConcepts.stream().map(c -> template.toBuilder().keyConcept(c).build()).collect(Collectors.toList()); } -} \ No newline at end of file +} diff --git a/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/SemanticAssetPathProcessorTest.java b/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/SemanticAssetPathProcessorTest.java index a16610ac..0324d4a7 100644 --- a/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/SemanticAssetPathProcessorTest.java +++ b/src/test/java/it/gov/innovazione/ndc/harvester/pathprocessors/SemanticAssetPathProcessorTest.java @@ -155,4 +155,4 @@ void ifModelCannotBeEnrichedShouldStopProcessingAndPropagate() { verifyNoInteractions(tripleStoreRepository); verifyNoInteractions(metadataRepository); } -} \ No newline at end of file +} diff --git a/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java b/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java index 447bb16e..7b2530b8 100644 --- a/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java +++ b/src/test/java/it/gov/innovazione/ndc/repository/SemanticAssetMetadataRepositoryTest.java @@ -1,15 +1,6 @@ package it.gov.innovazione.ndc.repository; -import static java.util.Objects.requireNonNull; -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - import it.gov.innovazione.ndc.harvester.model.index.SemanticAssetMetadata; - -import java.util.Optional; -import java.util.Set; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -27,6 +18,16 @@ import org.springframework.data.elasticsearch.core.query.ByQueryResponse; import org.springframework.data.elasticsearch.core.query.NativeSearchQuery; +import java.util.Collections; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + @ExtendWith(MockitoExtension.class) class SemanticAssetMetadataRepositoryTest { @Mock @@ -100,7 +101,7 @@ void shouldSearchUsingQueryStringAndFiltersAndPagination() { when(esOps.search(captor.capture(), any(Class.class))).thenReturn(searchHits); SearchPage searchResult = - repository.search("query", Set.of("TYPE1"), Set.of("THEME1"), PageRequest.of(0, 10)); + repository.search("query", Set.of("TYPE1"), Set.of("THEME1"), Collections.emptySet(), PageRequest.of(0, 10)); assertThat(searchResult.getSearchHits()).isEqualTo(searchHits); BoolQueryBuilder query = (BoolQueryBuilder) captor.getValue().getQuery(); @@ -128,7 +129,7 @@ void shouldSearchWithoutFiltersAndSearchText() { when(esOps.search(captor.capture(), any(Class.class))).thenReturn(searchHits); SearchPage searchResult = - repository.search("", Set.of(), Set.of(), PageRequest.of(0, 10)); + repository.search("", Set.of(), Set.of(), Collections.emptySet(), PageRequest.of(0, 10)); assertThat(searchResult.getSearchHits()).isEqualTo(searchHits); BoolQueryBuilder query = (BoolQueryBuilder) captor.getValue().getQuery(); @@ -138,4 +139,4 @@ void shouldSearchWithoutFiltersAndSearchText() { assertThat(query.filter().size()).isEqualTo(0); } -} \ No newline at end of file +} diff --git a/src/test/java/it/gov/innovazione/ndc/service/SemanticAssetSearchServiceTest.java b/src/test/java/it/gov/innovazione/ndc/service/SemanticAssetSearchServiceTest.java index 204e062f..6af38011 100644 --- a/src/test/java/it/gov/innovazione/ndc/service/SemanticAssetSearchServiceTest.java +++ b/src/test/java/it/gov/innovazione/ndc/service/SemanticAssetSearchServiceTest.java @@ -27,7 +27,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -57,7 +56,7 @@ void shouldGetSearchResultForTerm() { SemanticAssetMetadata expectedData2 = SemanticAssetMetadata.builder().iri("2").build(); Pageable pageable = Pageable.ofSize(10).withPage(0); - when(metadataRepository.search(any(), any(), any(), any())).thenReturn(searchPageMock); + when(metadataRepository.search(any(), any(), any(), any(), any())).thenReturn(searchPageMock); when(searchPageMock.getPageable()).thenReturn(PageRequest.of(1, 10)); when(searchPageMock.getTotalElements()).thenReturn(11L); when(searchPageMock.getContent()).thenReturn(List.of(searchHitMock, searchHitMock)); @@ -65,7 +64,7 @@ void shouldGetSearchResultForTerm() { SearchResult result = searchService.search("term", Set.of("ONTOLOGY", "SCHEMA"), - Set.of("EDUC", "AGRI"), pageable); + Set.of("EDUC", "AGRI"), Set.of(), pageable); assertThat(result.getTotalCount()).isEqualTo(11L); assertThat(result.getLimit()).isEqualTo(10); @@ -74,7 +73,7 @@ void shouldGetSearchResultForTerm() { assertThat(result.getData().stream().filter(e -> e.getAssetIri().equals("1"))).isNotNull(); assertThat(result.getData().stream().filter(e -> e.getAssetIri().equals("2"))).isNotNull(); verify(metadataRepository).search("term", Set.of("ONTOLOGY", "SCHEMA"), - Set.of("EDUC", "AGRI"), pageable); + Set.of("EDUC", "AGRI"), Collections.emptySet(), pageable); } @Test @@ -111,7 +110,7 @@ void shouldGetVocabularies() { .build(); Pageable pageable = Pageable.ofSize(10).withPage(0); - when(metadataRepository.search(any(), any(), any(), any())).thenReturn(searchPageMock); + when(metadataRepository.search(any(), any(), any(), any(), any())).thenReturn(searchPageMock); when(searchPageMock.getPageable()).thenReturn(PageRequest.of(1, 10)); when(searchPageMock.getTotalElements()).thenReturn(11L); when(searchPageMock.getContent()).thenReturn(List.of(searchHitMock, searchHitMock)); @@ -125,6 +124,6 @@ void shouldGetVocabularies() { assertThat(result.getData()).hasSize(2); assertThat(result.getData().stream().filter(e -> e.getTitle().equals("Some vocab"))).isNotNull(); assertThat(result.getData().stream().filter(e -> e.getTitle().equals("Some other vocab"))).isNotNull(); - verify(metadataRepository).search("", Set.of("CONTROLLED_VOCABULARY"), emptySet(), pageable); + verify(metadataRepository).search("", Set.of("CONTROLLED_VOCABULARY"), emptySet(), Collections.emptySet(), pageable); } -} \ No newline at end of file +}