Skip to content

Commit

Permalink
Merge pull request DSpace#8184 from 4Science/CST-5249
Browse files Browse the repository at this point in the history
Add a correction service to dspace to enhance the data quality (was Enrich local data via the OpenAIRE Graph)
  • Loading branch information
tdonohue authored Dec 18, 2023
2 parents 344d79b + 0a74a94 commit 4bf7d17
Show file tree
Hide file tree
Showing 120 changed files with 7,701 additions and 196 deletions.
2 changes: 2 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ services:
cp -r /opt/solr/server/solr/configsets/search/* search
precreate-core statistics /opt/solr/server/solr/configsets/statistics
cp -r /opt/solr/server/solr/configsets/statistics/* statistics
precreate-core qaevent /opt/solr/server/solr/configsets/qaevent
cp -r /opt/solr/server/solr/configsets/qaevent/* qaevent
exec solr -f
volumes:
assetstore:
Expand Down
6 changes: 6 additions & 0 deletions dspace-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -819,6 +819,12 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>eu.openaire</groupId>
<artifactId>broker-client</artifactId>
<version>1.1.2</version>
</dependency>

<dependency>
<groupId>org.mock-server</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1021,6 +1021,61 @@ private DiscoverResult retrieveCollectionsWithSubmit(Context context, DiscoverQu
return resp;
}

@Override
public Collection retrieveCollectionWithSubmitByEntityType(Context context, Item item,
String entityType) throws SQLException {
Collection ownCollection = item.getOwningCollection();
return retrieveWithSubmitCollectionByEntityType(context, ownCollection.getCommunities(), entityType);
}

private Collection retrieveWithSubmitCollectionByEntityType(Context context, List<Community> communities,
String entityType) {

for (Community community : communities) {
Collection collection = retrieveCollectionWithSubmitByCommunityAndEntityType(context, community,
entityType);
if (collection != null) {
return collection;
}
}

for (Community community : communities) {
List<Community> parentCommunities = community.getParentCommunities();
Collection collection = retrieveWithSubmitCollectionByEntityType(context, parentCommunities, entityType);
if (collection != null) {
return collection;
}
}

return retrieveCollectionWithSubmitByCommunityAndEntityType(context, null, entityType);
}

@Override
public Collection retrieveCollectionWithSubmitByCommunityAndEntityType(Context context, Community community,
String entityType) {
context.turnOffAuthorisationSystem();
List<Collection> collections;
try {
collections = findCollectionsWithSubmit(null, context, community, entityType, 0, 1);
} catch (SQLException | SearchServiceException e) {
throw new RuntimeException(e);
}
context.restoreAuthSystemState();
if (collections != null && collections.size() > 0) {
return collections.get(0);
}
if (community != null) {
for (Community subCommunity : community.getSubcommunities()) {
Collection collection = retrieveCollectionWithSubmitByCommunityAndEntityType(context,
subCommunity, entityType);
if (collection != null) {
return collection;
}
}
}
return null;
}

@Override
public List<Collection> findCollectionsWithSubmit(String q, Context context, Community community, String entityType,
int offset, int limit) throws SQLException, SearchServiceException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
import org.dspace.orcid.service.OrcidSynchronizationService;
import org.dspace.orcid.service.OrcidTokenService;
import org.dspace.profile.service.ResearcherProfileService;
import org.dspace.qaevent.dao.QAEventsDAO;
import org.dspace.services.ConfigurationService;
import org.dspace.versioning.service.VersioningService;
import org.dspace.workflow.WorkflowItemService;
Expand Down Expand Up @@ -170,6 +171,9 @@ public class ItemServiceImpl extends DSpaceObjectServiceImpl<Item> implements It
@Autowired(required = true)
protected SubscribeService subscribeService;

@Autowired
private QAEventsDAO qaEventsDao;

protected ItemServiceImpl() {
super();
}
Expand Down Expand Up @@ -819,6 +823,11 @@ protected void rawDelete(Context context, Item item) throws AuthorizeException,
orcidToken.setProfileItem(null);
}

List<QAEventProcessed> qaEvents = qaEventsDao.findByItem(context, item);
for (QAEventProcessed qaEvent : qaEvents) {
qaEventsDao.delete(context, qaEvent);
}

//Only clear collections after we have removed everything else from the item
item.clearCollections();
item.setOwningCollection(null);
Expand Down
213 changes: 213 additions & 0 deletions dspace-api/src/main/java/org/dspace/content/QAEvent.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.content;

import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Date;

import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import org.dspace.qaevent.service.dto.OpenaireMessageDTO;
import org.dspace.qaevent.service.dto.QAMessageDTO;
import org.dspace.util.RawJsonDeserializer;

/**
* This class represent the Quality Assurance broker data as loaded in our solr
* qaevent core
*
*/
public class QAEvent {
public static final char[] HEX_DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
'f' };
public static final String ACCEPTED = "accepted";
public static final String REJECTED = "rejected";
public static final String DISCARDED = "discarded";

public static final String OPENAIRE_SOURCE = "openaire";

private String source;

private String eventId;
/**
* contains the targeted dspace object,
* ie: oai:www.openstarts.units.it:123456789/1120 contains the handle
* of the DSpace pbject in its final part 123456789/1120
* */
private String originalId;

/**
* evaluated with the targeted dspace object id
*
* */
private String target;

private String related;

private String title;

private String topic;

private double trust;

@JsonDeserialize(using = RawJsonDeserializer.class)
private String message;

private Date lastUpdate;

private String status = "PENDING";

public QAEvent() {
}

public QAEvent(String source, String originalId, String target, String title,
String topic, double trust, String message, Date lastUpdate) {
super();
this.source = source;
this.originalId = originalId;
this.target = target;
this.title = title;
this.topic = topic;
this.trust = trust;
this.message = message;
this.lastUpdate = lastUpdate;
try {
computedEventId();
} catch (NoSuchAlgorithmException | UnsupportedEncodingException e) {
throw new IllegalStateException(e);
}

}

public String getOriginalId() {
return originalId;
}

public void setOriginalId(String originalId) {
this.originalId = originalId;
}

public String getTitle() {
return title;
}

public void setTitle(String title) {
this.title = title;
}

public String getTopic() {
return topic;
}

public void setTopic(String topic) {
this.topic = topic;
}

public double getTrust() {
return trust;
}

public void setTrust(double trust) {
this.trust = trust;
}

public String getMessage() {
return message;
}

public void setMessage(String message) {
this.message = message;
}

public String getEventId() {
if (eventId == null) {
try {
computedEventId();
} catch (NoSuchAlgorithmException | UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
return eventId;
}

public void setEventId(String eventId) {
this.eventId = eventId;
}

public String getTarget() {
return target;
}

public void setTarget(String target) {
this.target = target;
}

public Date getLastUpdate() {
return lastUpdate;
}

public void setLastUpdate(Date lastUpdate) {
this.lastUpdate = lastUpdate;
}

public void setRelated(String related) {
this.related = related;
}

public String getRelated() {
return related;
}

public void setStatus(String status) {
this.status = status;
}

public String getStatus() {
return status;
}

public String getSource() {
return source != null ? source : OPENAIRE_SOURCE;
}

public void setSource(String source) {
this.source = source;
}

/*
* DTO constructed via Jackson use empty constructor. In this case, the eventId
* must be compute on the get method. This method create a signature based on
* the event fields and store it in the eventid attribute.
*/
private void computedEventId() throws NoSuchAlgorithmException, UnsupportedEncodingException {
MessageDigest digester = MessageDigest.getInstance("MD5");
String dataToString = "source=" + source + ",originalId=" + originalId + ", title=" + title + ", topic="
+ topic + ", trust=" + trust + ", message=" + message;
digester.update(dataToString.getBytes("UTF-8"));
byte[] signature = digester.digest();
char[] arr = new char[signature.length << 1];
for (int i = 0; i < signature.length; i++) {
int b = signature[i];
int idx = i << 1;
arr[idx] = HEX_DIGITS[(b >> 4) & 0xf];
arr[idx + 1] = HEX_DIGITS[b & 0xf];
}
eventId = new String(arr);

}

public Class<? extends QAMessageDTO> getMessageDtoClass() {
switch (getSource()) {
case OPENAIRE_SOURCE:
return OpenaireMessageDTO.class;
default:
throw new IllegalArgumentException("Unknown event's source: " + getSource());
}
}

}
Loading

0 comments on commit 4bf7d17

Please sign in to comment.