Skip to content

Commit

Permalink
Merged in 2023_02_x-DQ-26 (pull request DSpace#1676)
Browse files Browse the repository at this point in the history 
DQ-26

Approved-by: Giuseppe Digilio
  • Loading branch information
@vins01-4science @atarix83
vins01-4science authored and atarix83 committed Feb 9, 2024
2 parents e666ff8 + f18a000 commit 908d6d6
Show file tree
Hide file tree
Showing 18 changed files with 497 additions and 95 deletions.
8 changes: 4 additions & 4 deletions dspace-api/src/main/java/org/dspace/app/deduplication/service/impl/SolrDedupServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@
import org.dspace.app.deduplication.service.DedupService;
import org.dspace.app.deduplication.service.SearchDeduplication;
import org.dspace.app.deduplication.service.SolrDedupServiceIndexPlugin;
import org.dspace.app.deduplication.utils.DedupUtils;
import org.dspace.app.deduplication.utils.DuplicateItemInfo;
import org.dspace.app.deduplication.utils.IDedupUtils;
import org.dspace.app.deduplication.utils.Signature;
import org.dspace.app.util.Util;
import org.dspace.authorize.AuthorizeException;
Expand Down Expand Up @@ -174,7 +174,7 @@ public class SolrDedupServiceImpl implements DedupService {
protected VersioningService versioningService;

@Autowired(required = true)
protected DedupUtils dedupUtils;
protected IDedupUtils dedupUtils;

/***
* Deduplication status
Expand Down Expand Up @@ -750,8 +750,8 @@ private void setDuplicateDecision(Context context, Item item, UUID duplicatedIte
private List<DuplicateItemInfo> findDuplicationWithDecisions(Context context, Item item) {
try {
return dedupUtils.getAdminDuplicateByIdAndType(context, item.getID(), item.getType()).stream()
.filter(duplication -> isNotEmpty(duplication.getDecisionTypes()))
.collect(Collectors.toList());
.filter(duplication -> isNotEmpty(duplication.getDecisionTypes()))
.collect(Collectors.toList());
} catch (SQLException | SearchServiceException e) {
throw new RuntimeException(e);
}
Expand Down
61 changes: 38 additions & 23 deletions dspace-api/src/main/java/org/dspace/app/deduplication/utils/DedupUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,15 @@
import org.dspace.services.ConfigurationService;
import org.dspace.util.ItemUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

/**
* Utility class used to search for duplicates inside the dedup solr core.
*
*/
public class DedupUtils {

@Service
public class DedupUtils implements IDedupUtils {

private static Logger log = LogManager.getLogger(DedupUtils.class);

Expand All @@ -64,11 +67,14 @@ public class DedupUtils {
@Autowired(required = true)
protected ConfigurationService configurationService;

public DuplicateInfoList findSignatureWithDuplicate(Context context, String signatureType, int resourceType,
int limit, int offset, int rule) throws SearchServiceException, SQLException {
@Override
public Collection<DuplicateInfo> findSignatureWithDuplicate(Context context, String signatureType, int resourceType,
int limit, int offset, int rule)
throws SearchServiceException, SQLException {
return findPotentialMatch(context, signatureType, resourceType, limit, offset, rule);
}

@Override
public Map<String, Integer> countSignaturesWithDuplicates(String query, int resourceTypeId)
throws SearchServiceException {
Map<String, Integer> results = new HashMap<String, Integer>();
Expand Down Expand Up @@ -113,6 +119,7 @@ public Map<String, Integer> countSignaturesWithDuplicates(String query, int reso
return results;
}

@Override
public Map<String, Integer> countSuggestedDuplicate(String query, int resourceTypeId)
throws SearchServiceException {
Map<String, Integer> results = new HashMap<String, Integer>();
Expand Down Expand Up @@ -241,8 +248,9 @@ private boolean hasStoredDecision(UUID firstItemID, UUID secondItemID, Duplicate
return !response.getResults().isEmpty();
}

@Override
public boolean matchExist(Context context, UUID itemID, UUID targetItemID, Integer resourceType,
String signatureType, Boolean isInWorkflow) throws SQLException, SearchServiceException {
String signatureType, Boolean isInWorkflow) throws SQLException, SearchServiceException {
boolean exist = false;
List<DuplicateItemInfo> potentialDuplicates = findDuplicate(context, itemID, resourceType, null, isInWorkflow);
for (DuplicateItemInfo match : potentialDuplicates) {
Expand All @@ -256,6 +264,7 @@ public boolean matchExist(Context context, UUID itemID, UUID targetItemID, Integ

}

@Override
public boolean rejectAdminDups(Context context, UUID firstId, UUID secondId, Integer type)
throws SQLException, AuthorizeException {
if (firstId == secondId) {
Expand Down Expand Up @@ -309,6 +318,7 @@ public boolean rejectAdminDups(Context context, UUID firstId, UUID secondId, Int
* @throws AuthorizeException
* @throws SearchServiceException
*/
@Override
public boolean rejectAdminDups(Context context, UUID itemID, String signatureType, int resourceType)
throws SQLException, AuthorizeException, SearchServiceException {

Expand Down Expand Up @@ -336,15 +346,17 @@ public boolean rejectAdminDups(Context context, UUID itemID, String signatureTyp

}

@Override
public void rejectAdminDups(Context context, List<DSpaceObject> items, String signatureID)
throws SQLException, AuthorizeException, SearchServiceException {
for (DSpaceObject item : items) {
rejectAdminDups(context, item.getID(), signatureID, item.getType());
}
}

@Override
public void verify(Context context, int dedupId, UUID firstId, UUID secondId, int type, boolean toFix, String note,
boolean check) throws SQLException, AuthorizeException {
boolean check) throws SQLException, AuthorizeException {
UUID[] sortedIds = new UUID[] { firstId, secondId };
Arrays.sort(sortedIds);
firstId = sortedIds[0];
Expand Down Expand Up @@ -417,8 +429,9 @@ private Deduplication retrieveDuplicationRow(Context context, UUID firstId, UUID
return row;
}

@Override
public void setDuplicateDecision(Context context, UUID firstId, UUID secondId, Integer type,
DuplicateDecisionObjectRest decisionObject)
DuplicateDecisionObjectRest decisionObject)
throws AuthorizeException, SQLException, SearchServiceException {

if (hasAuthorization(context, firstId, secondId)) {
Expand Down Expand Up @@ -478,6 +491,7 @@ public void setDuplicateDecision(Context context, UUID firstId, UUID secondId, I
}
}

@Override
public boolean validateDecision(DuplicateDecisionObjectRest decisionObject) {
boolean valid = false;

Expand All @@ -500,8 +514,9 @@ public boolean validateDecision(DuplicateDecisionObjectRest decisionObject) {
return valid;
}

@Override
public boolean rejectDups(Context context, UUID firstId, UUID secondId, Integer type, boolean notDupl, String note,
boolean check) throws SQLException {
boolean check) throws SQLException {
UUID[] sortedIds = new UUID[] { firstId, secondId };
Arrays.sort(sortedIds);
Deduplication row = null;
Expand Down Expand Up @@ -547,11 +562,9 @@ public boolean rejectDups(Context context, UUID firstId, UUID secondId, Integer
return false;
}

private DuplicateInfoList findPotentialMatch(Context context, String signatureType, int resourceType, int start,
private List<DuplicateInfo> findPotentialMatch(Context context, String signatureType, int resourceType, int start,
int rows, int rule) throws SearchServiceException, SQLException {

DuplicateInfoList dil = new DuplicateInfoList();

if (StringUtils.isNotEmpty(signatureType)) {
if (!StringUtils.contains(signatureType, "_signature")) {
signatureType += "_signature";
Expand Down Expand Up @@ -594,7 +607,7 @@ private DuplicateInfoList findPotentialMatch(Context context, String signatureTy

FacetField facetField = responseFacet.getFacetField(signatureType);

List<DuplicateInfo> result = new ArrayList<DuplicateInfo>();
List<DuplicateInfo> result = new ArrayList<>();

int index = 0;
for (Count facetHit : facetField.getValues()) {
Expand Down Expand Up @@ -653,10 +666,7 @@ private DuplicateInfoList findPotentialMatch(Context context, String signatureTy
}
index++;
}

dil.setDsi(result);
dil.setSize(facetField.getValues().size());
return dil;
return result;
}

private DuplicateSignatureInfo findPotentialMatchByID(Context context, String signatureType, int resourceType,
Expand Down Expand Up @@ -699,38 +709,45 @@ private DuplicateSignatureInfo findPotentialMatchByID(Context context, String si
return dsi;
}

@Override
public DedupService getDedupService() {
return dedupService;
}

@Override
public void setDedupService(DedupService dedupService) {
this.dedupService = dedupService;
}

@Override
public void commit() {
dedupService.commit();
}

@Override
public List<DuplicateItemInfo> getDuplicateByIDandType(Context context, UUID itemID, int typeID,
boolean isInWorkflow) throws SQLException, SearchServiceException {
boolean isInWorkflow)
throws SQLException, SearchServiceException {
return getDuplicateByIdAndTypeAndSignatureType(context, itemID, typeID, null, isInWorkflow);
}

@Override
public List<DuplicateItemInfo> getDuplicateByIdAndTypeAndSignatureType(Context context, UUID itemID, int typeID,
String signatureType, boolean isInWorkflow) throws SQLException, SearchServiceException {
String signatureType, boolean isInWorkflow)
throws SQLException, SearchServiceException {
return findDuplicate(context, itemID, typeID, signatureType, isInWorkflow);
}

@Override
public List<DuplicateItemInfo> getAdminDuplicateByIdAndType(Context context, UUID itemID, int typeID)
throws SQLException, SearchServiceException {
return findDuplicate(context, itemID, typeID, null, null);
}

public DuplicateInfoList findSuggestedDuplicate(Context context, int resourceType, int start, int rows)
@Override
public List<DuplicateInfo> findSuggestedDuplicate(Context context, int resourceType, int start, int rows)
throws SearchServiceException, SQLException {

DuplicateInfoList dil = new DuplicateInfoList();

SolrQuery solrQueryInternal = new SolrQuery();

solrQueryInternal.setQuery(SolrDedupServiceImpl.SUBQUERY_NOT_IN_REJECTED);
Expand Down Expand Up @@ -774,8 +791,6 @@ public DuplicateInfoList findSuggestedDuplicate(Context context, int resourceTyp
index++;
}

dil.setDsi(result);
dil.setSize(solrDocumentList.getNumFound());
return dil;
return result;
}
}
34 changes: 0 additions & 34 deletions dspace-api/src/main/java/org/dspace/app/deduplication/utils/DuplicateInfoList.java
View file

This file was deleted.

79 changes: 79 additions & 0 deletions dspace-api/src/main/java/org/dspace/app/deduplication/utils/IDedupUtils.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.app.deduplication.utils;

import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import org.dspace.app.deduplication.model.DuplicateDecisionObjectRest;
import org.dspace.app.deduplication.service.DedupService;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
import org.dspace.core.Context;
import org.dspace.discovery.SearchServiceException;

/**
* @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com)
**/
public interface IDedupUtils {
Collection<DuplicateInfo> findSignatureWithDuplicate(Context context, String signatureType, int resourceType,
int limit, int offset, int rule) throws SearchServiceException,
SQLException;

Map<String, Integer> countSignaturesWithDuplicates(String query, int resourceTypeId)
throws SearchServiceException;

Map<String, Integer> countSuggestedDuplicate(String query, int resourceTypeId)
throws SearchServiceException;

boolean matchExist(Context context, UUID itemID, UUID targetItemID, Integer resourceType,
String signatureType, Boolean isInWorkflow) throws SQLException, SearchServiceException;

boolean rejectAdminDups(Context context, UUID firstId, UUID secondId, Integer type)
throws SQLException, AuthorizeException;

boolean rejectAdminDups(Context context, UUID itemID, String signatureType, int resourceType)
throws SQLException, AuthorizeException, SearchServiceException;

void rejectAdminDups(Context context, List<DSpaceObject> items, String signatureID)
throws SQLException, AuthorizeException, SearchServiceException;

void verify(Context context, int dedupId, UUID firstId, UUID secondId, int type, boolean toFix, String note,
boolean check) throws SQLException, AuthorizeException;

void setDuplicateDecision(Context context, UUID firstId, UUID secondId, Integer type,
DuplicateDecisionObjectRest decisionObject)
throws AuthorizeException, SQLException, SearchServiceException;

boolean validateDecision(DuplicateDecisionObjectRest decisionObject);

boolean rejectDups(Context context, UUID firstId, UUID secondId, Integer type, boolean notDupl, String note,
boolean check) throws SQLException;

DedupService getDedupService();

void setDedupService(DedupService dedupService);

void commit();

List<DuplicateItemInfo> getDuplicateByIDandType(Context context, UUID itemID, int typeID,
boolean isInWorkflow) throws SQLException, SearchServiceException;

List<DuplicateItemInfo> getDuplicateByIdAndTypeAndSignatureType(Context context, UUID itemID, int typeID,
String signatureType, boolean isInWorkflow)
throws SQLException, SearchServiceException;

List<DuplicateItemInfo> getAdminDuplicateByIdAndType(Context context, UUID itemID, int typeID)
throws SQLException, SearchServiceException;

Collection<DuplicateInfo> findSuggestedDuplicate(Context context, int resourceType, int start, int rows)
throws SearchServiceException, SQLException;
}
11 changes: 6 additions & 5 deletions dspace-api/src/main/java/org/dspace/deduplication/service/DeduplicationService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.dspace.deduplication.Deduplication;

public interface DeduplicationService {

/**
* Create a new Deduplication object
*
Expand All @@ -23,7 +24,7 @@ public interface DeduplicationService {
* @throws SQLException An exception that provides information on a database
* access error or other errors.
*/
public Deduplication create(Context context, Deduplication dedup) throws SQLException;
Deduplication create(Context context, Deduplication dedup) throws SQLException;

/***
* Return all deduplication objects
Expand All @@ -35,7 +36,7 @@ public interface DeduplicationService {
* @throws SQLException An exception that provides information on a database
* access error or other errors.
*/
public List<Deduplication> findAll(Context context, int pageSize, int offset) throws SQLException;
List<Deduplication> findAll(Context context, int pageSize, int offset) throws SQLException;

/**
* Count all accounts.
Expand All @@ -55,11 +56,11 @@ public interface DeduplicationService {
* @throws SQLException An exception that provides information on a database
* access error or other errors.
*/
public void update(Context context, Deduplication dedup) throws SQLException;
void update(Context context, Deduplication dedup) throws SQLException;

public List<Deduplication> getDeduplicationByFirstAndSecond(Context context, UUID firstId, UUID secondId)
List<Deduplication> getDeduplicationByFirstAndSecond(Context context, UUID firstId, UUID secondId)
throws SQLException;

public Deduplication uniqueDeduplicationByFirstAndSecond(Context context, UUID firstId, UUID secondId)
Deduplication uniqueDeduplicationByFirstAndSecond(Context context, UUID firstId, UUID secondId)
throws SQLException;
}
Loading

0 comments on commit 908d6d6

Please sign in to comment.