diff --git a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java index 6b3ef003edca..b34e8bdfbd7c 100644 --- a/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java @@ -1979,7 +1979,9 @@ public List getMetadata(Item item, String schema, String element, List dbMetadataValues = item.getMetadata(); List fullMetadataValueList = new LinkedList<>(); - fullMetadataValueList.addAll(relationshipMetadataService.getRelationshipMetadata(item, true)); + if (configurationService.getBooleanProperty("item.enable-virtual-metadata", false)) { + fullMetadataValueList.addAll(relationshipMetadataService.getRelationshipMetadata(item, true)); + } fullMetadataValueList.addAll(dbMetadataValues); item.setCachedMetadata(MetadataValueComparators.sort(fullMetadataValueList)); diff --git a/dspace-api/src/main/java/org/dspace/content/dao/ItemForMetadataEnhancementUpdateDAO.java b/dspace-api/src/main/java/org/dspace/content/dao/ItemForMetadataEnhancementUpdateDAO.java new file mode 100644 index 000000000000..532e925a965e --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/dao/ItemForMetadataEnhancementUpdateDAO.java @@ -0,0 +1,49 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.dao; + +import java.sql.SQLException; +import java.util.UUID; + +import org.dspace.core.Context; + +public interface ItemForMetadataEnhancementUpdateDAO { + + /** + * Add to the metadata_enhancement_update table queue an entry of each items + * that is potentially affected by the update of the Item with the specified + * uuid. The items potentially affected are the one that have the provided uuid + * as value of a cris.virtualsource.* metadata + * + * @param context the DSpace Context object + * @param uuid the uuid of the updated item + * @return the number of affected items scheduled for update + * @throws SQLException if a problem with the database occurs + */ + int saveAffectedItemsForUpdate(Context context, UUID uuid); + + /** + * Remove from the metadata_enhancement_update table queue the entry if any + * related to the specified id in older than the current date + * + * @param context the DSpace Context object + * @param itemToRemove the uuid of the processed item + * @throws SQLException if a problem with the database occurs + */ + void removeItemForUpdate(Context context, UUID itemToRemove); + + /** + * Extract and remove from the table the first uuid to process from the + * itemupdate_metadata_enhancement table ordered by date queued asc (older + * first) + * + * @param context + * @return + */ + UUID pollItemToUpdate(Context context); +} diff --git a/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemForMetadataEnhancementUpdateDAOImpl.java b/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemForMetadataEnhancementUpdateDAOImpl.java new file mode 100644 index 000000000000..4d295f938633 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/dao/impl/ItemForMetadataEnhancementUpdateDAOImpl.java @@ -0,0 +1,152 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.dao.impl; + +import java.sql.SQLException; +import java.util.UUID; + +import org.dspace.content.MetadataSchema; +import org.dspace.content.dao.ItemForMetadataEnhancementUpdateDAO; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.MetadataSchemaService; +import org.dspace.core.Context; +import org.dspace.core.DBConnection; +import org.dspace.services.ConfigurationService; +import org.dspace.utils.DSpace; +import org.hibernate.Session; +import org.hibernate.query.NativeQuery; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Hibernate implementation of the Database Access Object interface class for + * the ItemForMetadataEnhancementUpdate object. This class is responsible for + * all database calls for the ItemForMetadataEnhancementUpdate object and is + * autowired by spring This class should never be accessed directly. + */ +public class ItemForMetadataEnhancementUpdateDAOImpl implements ItemForMetadataEnhancementUpdateDAO { + @Autowired + ConfigurationService configurationService; + + @Override + public void removeItemForUpdate(Context context, UUID itemToRemove) { + try { + Session session = getHibernateSession(); + String sql = "DELETE FROM itemupdate_metadata_enhancement WHERE uuid = :uuid"; + NativeQuery query = session.createNativeQuery(sql); + query.setParameter("uuid", itemToRemove); + query.executeUpdate(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + @Override + public UUID pollItemToUpdate(Context context) { + try { + Session session = getHibernateSession(); + String sql = "SELECT cast(uuid as varchar) FROM itemupdate_metadata_enhancement" + + " ORDER BY date_queued ASC LIMIT 1"; + NativeQuery query = session.createNativeQuery(sql); + Object uuidObj = query.uniqueResult(); + if (uuidObj != null) { + UUID uuid; + if (uuidObj instanceof String) { + uuid = (UUID) UUID.fromString((String) uuidObj); + } else { + throw new RuntimeException("Unexpected result type from the database " + uuidObj); + } + removeItemForUpdate(context, uuid); + return uuid; + } else { + return null; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + + } + + @Override + public int saveAffectedItemsForUpdate(Context context, UUID uuid) { + try { + Session session = getHibernateSession(); + MetadataSchemaService schemaService = ContentServiceFactory.getInstance().getMetadataSchemaService(); + MetadataSchema schema = schemaService.find(context, "cris"); + String sqlInsertOrUpdate; + if ("org.h2.Driver".equals(configurationService.getProperty("db.driver"))) { + // H2 doesn't support the INSERT OR UPDATE statement so let's do in two steps + // update queued date for records already in the queue + String sqlUpdate = "UPDATE itemupdate_metadata_enhancement" + + " SET date_queued = CURRENT_TIMESTAMP" + + " WHERE uuid IN (" + + " SELECT dspace_object_id FROM metadatavalue " + " WHERE metadata_field_id IN" + + " ( SELECT metadata_field_id FROM metadatafieldregistry " + + " WHERE metadata_schema_id = :schema AND element = 'virtualsource')" + + " AND text_value = :uuid)"; + String sqlInsert = + "INSERT INTO itemupdate_metadata_enhancement (uuid, date_queued)" + + " SELECT DISTINCT dspace_object_id, CURRENT_TIMESTAMP FROM metadatavalue " + + " WHERE metadata_field_id IN" + + " ( SELECT metadata_field_id FROM metadatafieldregistry " + + " WHERE metadata_schema_id = :schema AND element = 'virtualsource'" + + " AND text_value = :uuid) " + + " AND dspace_object_id NOT IN (" + + " SELECT uuid" + + " FROM itemupdate_metadata_enhancement" + + " )"; + NativeQuery queryUpdate = session.createNativeQuery(sqlUpdate); + queryUpdate.setParameter("uuid", uuid.toString()); + queryUpdate.setParameter("schema", schema.getID()); + queryUpdate.executeUpdate(); + NativeQuery queryInsert = session.createNativeQuery(sqlInsert); + queryInsert.setParameter("uuid", uuid.toString()); + queryInsert.setParameter("schema", schema.getID()); + return queryInsert.executeUpdate(); + } else { + sqlInsertOrUpdate = "INSERT INTO itemupdate_metadata_enhancement (uuid, date_queued) " + + " SELECT DISTINCT dspace_object_id, CURRENT_TIMESTAMP FROM metadatavalue " + + " WHERE metadata_field_id IN" + + " ( SELECT metadata_field_id FROM metadatafieldregistry " + + " WHERE metadata_schema_id = :schema AND element = 'virtualsource'" + + " AND text_value = :uuid) " + + " ON CONFLICT (uuid) DO UPDATE" + + " SET date_queued = EXCLUDED.date_queued"; + NativeQuery queryInsertOrUpdate = session.createNativeQuery(sqlInsertOrUpdate); + queryInsertOrUpdate.setParameter("uuid", uuid.toString()); + queryInsertOrUpdate.setParameter("schema", schema.getID()); + return queryInsertOrUpdate.executeUpdate(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * The Hibernate Session used in the current thread + * + * @return the current Session. + * @throws SQLException + */ + private Session getHibernateSession() throws SQLException { + DBConnection dbConnection = new DSpace().getServiceManager().getServiceByName(null, DBConnection.class); + return ((Session) dbConnection.getSession()); + } + + public UUID ConvertByteArrayToUUID(byte[] bytea) { + long mostSigBits = 0; + long leastSigBits = 0; + for (int i = 0; i < 8; i++) { + mostSigBits = (mostSigBits << 8) | (bytea[i] & 0xff); + leastSigBits = (leastSigBits << 8) | (bytea[i + 8] & 0xff); + } + + UUID uuid = new UUID(mostSigBits, leastSigBits); + return uuid; + } +} \ No newline at end of file diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumer.java index c526537bf5ac..ebac7a82e0ad 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumer.java @@ -9,9 +9,12 @@ import java.util.HashSet; import java.util.Set; +import java.util.UUID; import org.dspace.content.Item; import org.dspace.content.enhancer.service.ItemEnhancerService; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; import org.dspace.core.Context; import org.dspace.event.Consumer; import org.dspace.event.Event; @@ -29,10 +32,12 @@ public class ItemEnhancerConsumer implements Consumer { public static final String ITEMENHANCER_ENABLED = "itemenhancer.enabled"; - private Set itemsAlreadyProcessed = new HashSet(); + private Set itemsToProcess = new HashSet(); private ItemEnhancerService itemEnhancerService; + private ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + private ConfigurationService configurationService = DSpaceServicesFactory.getInstance().getConfigurationService(); @Override @@ -53,19 +58,11 @@ public void consume(Context context, Event event) throws Exception { } Item item = (Item) event.getSubject(context); - if (item == null || itemsAlreadyProcessed.contains(item) || !item.isArchived()) { + if (item == null || !item.isArchived()) { return; } - itemsAlreadyProcessed.add(item); - - context.turnOffAuthorisationSystem(); - try { - itemEnhancerService.enhance(context, item, false); - } finally { - context.restoreAuthSystemState(); - } - + itemsToProcess.add(item.getID()); } protected boolean isConsumerEnabled() { @@ -74,7 +71,19 @@ protected boolean isConsumerEnabled() { @Override public void end(Context ctx) throws Exception { - itemsAlreadyProcessed.clear(); + ctx.turnOffAuthorisationSystem(); + try { + for (UUID uuid : itemsToProcess) { + Item item = itemService.find(ctx, uuid); + if (item != null) { + itemEnhancerService.enhance(ctx, item, false); + itemEnhancerService.saveAffectedItemsForUpdate(ctx, item.getID()); + } + } + } finally { + ctx.restoreAuthSystemState(); + } + itemsToProcess.clear(); } } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancer.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancer.java index 7a6f2927092c..870dfbddfa8e 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancer.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancer.java @@ -21,6 +21,7 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.dspace.authority.service.AuthorityValueService; import org.dspace.content.Item; import org.dspace.content.MetadataValue; import org.dspace.content.authority.Choices; @@ -49,10 +50,13 @@ public class RelatedEntityItemEnhancer extends AbstractItemEnhancer { @Autowired private ItemService itemService; + @Autowired + private RelatedEntityItemEnhancerUtils relatedEntityItemEnhancerUtils; + /** - * the entity that can be extended by this enhancer, i.e. Publication + * the entities that can be extended by this enhancer, i.e. Publication */ - private String sourceEntityType; + private List sourceEntityTypes; /** * the metadata used to navigate the relation, i.e. dc.contributor.author @@ -66,7 +70,7 @@ public class RelatedEntityItemEnhancer extends AbstractItemEnhancer { @Override public boolean canEnhance(Context context, Item item) { - return sourceEntityType == null || sourceEntityType.equals(itemService.getEntityTypeLabel(item)); + return sourceEntityTypes == null || sourceEntityTypes.contains(itemService.getEntityTypeLabel(item)); } @Override @@ -81,7 +85,8 @@ public boolean enhance(Context context, Item item, boolean deepMode) { throw new SQLRuntimeException(e); } } else { - Map> currMetadataValues = getCurrentVirtualsMap(item); + Map> currMetadataValues = relatedEntityItemEnhancerUtils + .getCurrentVirtualsMap(item, getVirtualQualifier()); Map> toBeMetadataValues = getToBeVirtualMetadata(context, item); if (!equivalent(currMetadataValues, toBeMetadataValues)) { try { @@ -106,8 +111,8 @@ private void clearAllVirtualMetadata(Context context, Item item) throws SQLExcep private void addMetadata(Context context, Item item, Map> toBeMetadataValues) throws SQLException { for (Entry> metadataValues : toBeMetadataValues.entrySet()) { - addVirtualSourceField(context, item, metadataValues.getKey()); for (MetadataValueDTO dto : metadataValues.getValue()) { + addVirtualSourceField(context, item, metadataValues.getKey()); addVirtualField(context, item, dto.getValue(), dto.getAuthority(), dto.getLanguage(), dto.getConfidence()); } @@ -212,7 +217,8 @@ private boolean cleanObsoleteVirtualFields(Context context, Item item) throws SQ private List getObsoleteVirtualFields(Item item) { List obsoleteVirtualFields = new ArrayList<>(); - Map> currentVirtualsMap = getCurrentVirtualsMap(item); + Map> currentVirtualsMap = relatedEntityItemEnhancerUtils + .getCurrentVirtualsMap(item, getVirtualQualifier()); Set virtualSources = getVirtualSources(item); for (String authority : currentVirtualsMap.keySet()) { if (!virtualSources.contains(authority)) { @@ -235,41 +241,6 @@ private Set getVirtualSources(Item item) { .collect(Collectors.toSet()); } - private Map> getCurrentVirtualsMap(Item item) { - Map> currentVirtualsMap = new HashMap>(); - List sources = itemService.getMetadata(item, VIRTUAL_METADATA_SCHEMA, - VIRTUAL_SOURCE_METADATA_ELEMENT, getVirtualQualifier(), Item.ANY); - List generated = itemService.getMetadata(item, VIRTUAL_METADATA_SCHEMA, VIRTUAL_METADATA_ELEMENT, - getVirtualQualifier(), Item.ANY); - - if (sources.size() != generated.size()) { - LOGGER.error( - "inconsistent virtual metadata for the item {} got {} sources and {} generated virtual metadata", - item.getID().toString(), sources.size(), generated.size()); - } - - for (int i = 0; i < Integer.max(sources.size(), generated.size()); i++) { - String authority; - if (i < sources.size()) { - authority = sources.get(i).getValue(); - } else { - // we have less source than virtual metadata let's generate a random uuid to - // associate with these extra metadata so that they will be managed as obsolete - // value - authority = UUID.randomUUID().toString(); - } - List mvalues = currentVirtualsMap.get(authority); - if (mvalues == null) { - mvalues = new ArrayList(); - } - if (i < generated.size()) { - mvalues.add(generated.get(i)); - } - currentVirtualsMap.put(authority, mvalues); - } - return currentVirtualsMap; - } - private Optional getRelatedVirtualField(Item item, int pos) { return getVirtualFields(item).stream() .skip(pos) @@ -278,7 +249,8 @@ private Optional getRelatedVirtualField(Item item, int pos) { private boolean performEnhancement(Context context, Item item) throws SQLException { boolean result = false; - Map> currentVirtualsMap = getCurrentVirtualsMap(item); + Map> currentVirtualsMap = relatedEntityItemEnhancerUtils + .getCurrentVirtualsMap(item, getVirtualQualifier()); Set virtualSources = getVirtualSources(item); for (String authority : virtualSources) { boolean foundAtLeastOne = false; @@ -336,8 +308,14 @@ private List getVirtualFields(Item item) { private void addVirtualField(Context context, Item item, String value, String authority, String lang, int confidence) throws SQLException { - itemService.addMetadata(context, item, VIRTUAL_METADATA_SCHEMA, VIRTUAL_METADATA_ELEMENT, getVirtualQualifier(), - lang, value, authority, confidence); + if (StringUtils.startsWith(authority, AuthorityValueService.GENERATE) + || StringUtils.startsWith(authority, AuthorityValueService.REFERENCE)) { + itemService.addMetadata(context, item, VIRTUAL_METADATA_SCHEMA, VIRTUAL_METADATA_ELEMENT, + getVirtualQualifier(), lang, value, null, Choices.CF_UNSET); + } else { + itemService.addMetadata(context, item, VIRTUAL_METADATA_SCHEMA, VIRTUAL_METADATA_ELEMENT, + getVirtualQualifier(), lang, value, authority, confidence); + } } private void addVirtualSourceField(Context context, Item item, String sourceValueAuthority) throws SQLException { @@ -345,24 +323,8 @@ private void addVirtualSourceField(Context context, Item item, String sourceValu getVirtualQualifier(), null, sourceValueAuthority); } - public void setSourceEntityType(String sourceEntityType) { - this.sourceEntityType = sourceEntityType; - } - - @Deprecated - public void setSourceItemMetadataField(String sourceItemMetadataField) { - LOGGER.warn( - "RelatedEntityItemEnhancer configured using the old single source item metadata field, " - + "please update the configuration to use the list"); - this.sourceItemMetadataFields = List.of(sourceItemMetadataField); - } - - @Deprecated - public void setRelatedItemMetadataField(String relatedItemMetadataField) { - LOGGER.warn( - "RelatedEntityItemEnhancer configured using the old single related item metadata field, " - + "please update the configuration to use the list"); - this.relatedItemMetadataFields = List.of(relatedItemMetadataField); + public void setSourceEntityTypes(List sourceEntityTypes) { + this.sourceEntityTypes = sourceEntityTypes; } public void setRelatedItemMetadataFields(List relatedItemMetadataFields) { diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancerUtils.java b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancerUtils.java new file mode 100644 index 000000000000..064223b56422 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/impl/RelatedEntityItemEnhancerUtils.java @@ -0,0 +1,72 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer.impl; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.service.ItemService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Utility methods used by {@link RelatedEntityItemEnhancer} + * + * @author Andrea Bollini (andrea.bollini at 4science.com) + * + */ +public class RelatedEntityItemEnhancerUtils { + + @Autowired + private ItemService itemService; + + private static final Logger LOGGER = LoggerFactory.getLogger(RelatedEntityItemEnhancerUtils.class); + + public Map> getCurrentVirtualsMap(Item item, String virtualQualifier) { + Map> currentVirtualsMap = new HashMap>(); + List sources = itemService.getMetadata(item, RelatedEntityItemEnhancer.VIRTUAL_METADATA_SCHEMA, + RelatedEntityItemEnhancer.VIRTUAL_SOURCE_METADATA_ELEMENT, virtualQualifier, Item.ANY); + List generated = itemService.getMetadata(item, + RelatedEntityItemEnhancer.VIRTUAL_METADATA_SCHEMA, RelatedEntityItemEnhancer.VIRTUAL_METADATA_ELEMENT, + virtualQualifier, Item.ANY); + + if (sources.size() != generated.size()) { + LOGGER.error( + "inconsistent virtual metadata for the item {} got {} sources and {} generated virtual metadata", + item.getID().toString(), sources.size(), generated.size()); + } + + for (int i = 0; i < Integer.max(sources.size(), generated.size()); i++) { + String authority; + if (i < sources.size()) { + authority = sources.get(i).getValue(); + } else { + // we have less source than virtual metadata let's generate a random uuid to + // associate with these extra metadata so that they will be managed as obsolete + // value + authority = UUID.randomUUID().toString(); + } + List mvalues = currentVirtualsMap.get(authority); + if (mvalues == null) { + mvalues = new ArrayList(); + } + if (i < generated.size()) { + mvalues.add(generated.get(i)); + } + currentVirtualsMap.put(authority, mvalues); + } + return currentVirtualsMap; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java b/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java index 08170448e681..d345c55e82f5 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java @@ -7,7 +7,11 @@ */ package org.dspace.content.enhancer.service; +import java.sql.SQLException; +import java.util.UUID; + import org.dspace.content.Item; +import org.dspace.content.dao.ItemForMetadataEnhancementUpdateDAO; import org.dspace.core.Context; /** @@ -29,4 +33,21 @@ public interface ItemEnhancerService { */ void enhance(Context context, Item item, boolean deepMode); + /** + * Find items that could be affected by a change of the item with given uuid + * and save them to db for future processing + * + * @param context the DSpace Context + * @param uuid UUID of the changed item + */ + void saveAffectedItemsForUpdate(Context context, UUID uuid) throws SQLException; + + /** + * Extract the first uuid in the itemupdate_metadata_enhancement table, see + * {@link ItemForMetadataEnhancementUpdateDAO#pollItemToUpdate(Context)} + * + * @param context the DSpace Context + * @return UUID of the older item queued for update + */ + UUID pollItemToUpdate(Context context); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/service/impl/ItemEnhancerServiceImpl.java b/dspace-api/src/main/java/org/dspace/content/enhancer/service/impl/ItemEnhancerServiceImpl.java index e751a431ac37..97816515ea36 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/service/impl/ItemEnhancerServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/service/impl/ItemEnhancerServiceImpl.java @@ -9,9 +9,11 @@ import java.sql.SQLException; import java.util.List; +import java.util.UUID; import org.dspace.authorize.AuthorizeException; import org.dspace.content.Item; +import org.dspace.content.dao.ItemForMetadataEnhancementUpdateDAO; import org.dspace.content.enhancer.ItemEnhancer; import org.dspace.content.enhancer.service.ItemEnhancerService; import org.dspace.content.service.ItemService; @@ -32,10 +34,15 @@ public class ItemEnhancerServiceImpl implements ItemEnhancerService { @Autowired private ItemService itemService; + @Autowired + private ItemForMetadataEnhancementUpdateDAO itemForMetadataEnhancementUpdateDAO; + @Override public void enhance(Context context, Item item, boolean deepMode) { boolean isUpdateNeeded = false; - + if (deepMode) { + itemForMetadataEnhancementUpdateDAO.removeItemForUpdate(context, item.getID()); + } for (ItemEnhancer itemEnhancer : itemEnhancers) { if (itemEnhancer.canEnhance(context, item)) { isUpdateNeeded = itemEnhancer.enhance(context, item, deepMode) || isUpdateNeeded; @@ -44,9 +51,24 @@ public void enhance(Context context, Item item, boolean deepMode) { if (isUpdateNeeded) { updateItem(context, item); + try { + saveAffectedItemsForUpdate(context, item.getID()); + } catch (SQLException e) { + throw new RuntimeException(e.getMessage(), e); + } } } + @Override + public void saveAffectedItemsForUpdate(Context context, UUID uuid) throws SQLException { + itemForMetadataEnhancementUpdateDAO.saveAffectedItemsForUpdate(context, uuid); + } + + @Override + public UUID pollItemToUpdate(Context context) { + return itemForMetadataEnhancementUpdateDAO.pollItemToUpdate(context); + } + private void updateItem(Context context, Item item) { try { itemService.update(context, item); diff --git a/dspace-api/src/main/java/org/dspace/content/integration/crosswalks/virtualfields/VirtualFieldToEnhancedMetadata.java b/dspace-api/src/main/java/org/dspace/content/integration/crosswalks/virtualfields/VirtualFieldToEnhancedMetadata.java new file mode 100644 index 000000000000..2ea11f6c4ea8 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/content/integration/crosswalks/virtualfields/VirtualFieldToEnhancedMetadata.java @@ -0,0 +1,62 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.integration.crosswalks.virtualfields; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.enhancer.impl.RelatedEntityItemEnhancerUtils; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.core.Context; +import org.dspace.core.CrisConstants; +import org.springframework.beans.factory.annotation.Autowired; + +/** + * Implementation of {@link VirtualField} that returns the values from the a + * cris.virtual. metadata using the provided in the form of + * -- as source metadata. + * Source metadata that are not found in the cris.virtualsource. leads to a PLACEHOLDER + * + * @author Andrea Bollini at 4science.comm + * + */ +public class VirtualFieldToEnhancedMetadata implements VirtualField { + + @Autowired + private ItemService itemService; + + @Autowired + private RelatedEntityItemEnhancerUtils relatedEntityItemEnhancerUtils; + + @Override + public String[] getMetadata(Context context, Item item, String fieldName) { + ItemService itemService = ContentServiceFactory.getInstance().getItemService(); + String[] fieldBits = fieldName.split("\\."); + if (fieldBits.length != 3) { + throw new IllegalArgumentException( + "VirtualFieldToEnhancedMetadata must be used specifying the EnhancedMetadata qualifier as " + + "element and the source metadata as qualifier, i.e. virtual.department.dc-contributor-author"); + } + String virtualQualifier = fieldBits[1]; + String metadata = fieldBits[2].replaceAll("-", "."); + Map> map = relatedEntityItemEnhancerUtils.getCurrentVirtualsMap(item, + virtualQualifier); + List values = itemService.getMetadataByMetadataString(item, metadata).stream() + .map(mv -> mv.getAuthority() != null && map.containsKey(mv.getAuthority()) + ? map.get(mv.getAuthority()).get(0).getValue() + : CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE) + .collect(Collectors.toList()); + String[] resultValues = values.toArray(new String[0]); + return resultValues; + } + +} diff --git a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java index 9897639f04a6..7ffd7ecd004d 100644 --- a/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java +++ b/dspace-api/src/main/java/org/dspace/external/provider/impl/LiveImportDataProvider.java @@ -92,7 +92,7 @@ public void setDisplayMetadata(String displayMetadata) { public Optional getExternalDataObject(String id) { try { ExternalDataObject externalDataObject = getExternalDataObject(querySource.getRecord(id)); - return Optional.of(externalDataObject); + return Optional.ofNullable(externalDataObject); } catch (MetadataSourceException e) { throw new RuntimeException( "The live import provider " + querySource.getImportSource() + " throws an exception", e); diff --git a/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java b/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java index 1d6a6783018c..c5614bd1b98c 100644 --- a/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java +++ b/dspace-api/src/main/java/org/dspace/iiif/consumer/IIIFCacheEventConsumer.java @@ -125,9 +125,13 @@ public void consume(Context ctx, Event event) throws Exception { private void addToCacheEviction(DSpaceObject subject, DSpaceObject subject2, int type) { if (type == Constants.BITSTREAM) { - toEvictFromCanvasCache.add(subject2); + if (subject2 != null) { + toEvictFromCanvasCache.add(subject2); + } + } + if (subject != null) { + toEvictFromManifestCache.add(subject); } - toEvictFromManifestCache.add(subject); } @Override diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.03.07__create_table_items_for_update.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.03.07__create_table_items_for_update.sql new file mode 100644 index 000000000000..b1dfe8afc79a --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/h2/V7.6_2024.03.07__create_table_items_for_update.sql @@ -0,0 +1,18 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +----------------------------------------------------------------------------------- +-- Create TABLE itemupdate_metadata_enhancement +----------------------------------------------------------------------------------- + +CREATE TABLE itemupdate_metadata_enhancement +( + uuid UUID NOT NULL PRIMARY KEY, + date_queued TIMESTAMP NOT NULL +); +CREATE INDEX idx_date_queued ON itemupdate_metadata_enhancement(date_queued); diff --git a/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.03.07__create_table_items_for_update.sql b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.03.07__create_table_items_for_update.sql new file mode 100644 index 000000000000..b1dfe8afc79a --- /dev/null +++ b/dspace-api/src/main/resources/org/dspace/storage/rdbms/sqlmigration/postgres/V7.6_2024.03.07__create_table_items_for_update.sql @@ -0,0 +1,18 @@ +-- +-- The contents of this file are subject to the license and copyright +-- detailed in the LICENSE and NOTICE files at the root of the source +-- tree and available online at +-- +-- http://www.dspace.org/license/ +-- + +----------------------------------------------------------------------------------- +-- Create TABLE itemupdate_metadata_enhancement +----------------------------------------------------------------------------------- + +CREATE TABLE itemupdate_metadata_enhancement +( + uuid UUID NOT NULL PRIMARY KEY, + date_queued TIMESTAMP NOT NULL +); +CREATE INDEX idx_date_queued ON itemupdate_metadata_enhancement(date_queued); diff --git a/dspace-api/src/test/data/dspaceFolder/config/local.cfg b/dspace-api/src/test/data/dspaceFolder/config/local.cfg index b02b9fd15508..b7b020b7d4ee 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/local.cfg +++ b/dspace-api/src/test/data/dspaceFolder/config/local.cfg @@ -227,4 +227,6 @@ authority.controlled.dspace.object.owner = true # force the event system to work synchronously during test system-event.thread.size = 0 -vocabulary.plugin.srsc-noauthority.authority.store = false \ No newline at end of file +vocabulary.plugin.srsc-noauthority.authority.store = false +# disable the item enhancer poller during test +related-item-enhancer-poller.delay = -1 \ No newline at end of file diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/extra-metadata-enhancers-for-test.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/extra-metadata-enhancers-for-test.xml index 0311d8a26aa5..a713d3acbcc6 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/extra-metadata-enhancers-for-test.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/extra-metadata-enhancers-for-test.xml @@ -21,7 +21,11 @@ - + + + TestEntity + + dc.contributor.author diff --git a/dspace-api/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java b/dspace-api/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java index 55ceb779ba0b..1566f965a8d1 100644 --- a/dspace-api/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java +++ b/dspace-api/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java @@ -96,6 +96,10 @@ public static MetadataValueMatcher withNoPlace(String field, String value) { return with(field, value, null, null, null, -1); } + public static MetadataValueMatcher withNoPlace(String field, String value, String authority) { + return with(field, value, null, authority, null, 600); + } + public static MetadataValueMatcher withSecurity(String field, String value, Integer securityLevel) { return with(field, value, null, null, 0, -1, securityLevel); } diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumerIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumerIT.java index b2b34c1074fb..bd93c506d858 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumerIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/consumer/ItemEnhancerConsumerIT.java @@ -379,8 +379,8 @@ public void testEnhancementAfterItemUpdate() throws Exception { assertThat(getMetadataValues(publication, "cris.virtual.orcid"), contains( with("cris.virtual.orcid", "0000-0000-1111-2222"))); - assertThat(getMetadataValues(publication, "cris.virtualsource.orcid"), contains( - with("cris.virtualsource.orcid", personId))); + assertThat(getMetadataValues(publication, "cris.virtualsource.orcid"), contains( + with("cris.virtualsource.orcid", personId))); } diff --git a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java index 6d67c67a10ba..57d296f53fe9 100644 --- a/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java +++ b/dspace-api/src/test/java/org/dspace/content/enhancer/script/ItemEnhancerScriptIT.java @@ -295,25 +295,26 @@ public void testItemEnhancementWithForce() throws Exception { context.turnOffAuthorisationSystem(); - Item firstAuthor = ItemBuilder.createItem(context, collection) + Item editor = ItemBuilder.createItem(context, collection) .withTitle("Walter White") .withPersonMainAffiliation("4Science") .build(); - String firstAuthorId = firstAuthor.getID().toString(); + String editorId = editor.getID().toString(); - Item secondAuthor = ItemBuilder.createItem(context, collection) + Item author = ItemBuilder.createItem(context, collection) .withTitle("Jesse Pinkman") .withPersonMainAffiliation("Company") + .withPersonMainAffiliation("Another Company") .build(); - String secondAuthorId = secondAuthor.getID().toString(); + String authorId = author.getID().toString(); Item publication = ItemBuilder.createItem(context, collection) .withTitle("Test publication 2 ") .withEntityType("Publication") - .withEditor("Walter White", firstAuthorId) - .withAuthor("Jesse Pinkman", secondAuthorId) + .withEditor("Walter White", editorId) + .withAuthor("Jesse Pinkman", authorId) .build(); context.commit(); @@ -329,22 +330,24 @@ public void testItemEnhancementWithForce() throws Exception { publication = reload(publication); - assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(2)); - assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(2)); + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(3)); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(3)); assertThat(getMetadataValues(publication, "cris.virtual.department"), containsInAnyOrder( withNoPlace("cris.virtual.department", "4Science"), + withNoPlace("cris.virtual.department", "Another Company"), withNoPlace("cris.virtual.department", "Company"))); assertThat(getMetadataValues(publication, "cris.virtualsource.department"), containsInAnyOrder( - withNoPlace("cris.virtualsource.department", firstAuthorId), - withNoPlace("cris.virtualsource.department", secondAuthorId))); + withNoPlace("cris.virtualsource.department", editorId), + withNoPlace("cris.virtualsource.department", authorId), + withNoPlace("cris.virtualsource.department", authorId))); context.turnOffAuthorisationSystem(); MetadataValue authorToRemove = getMetadataValues(publication, "dc.contributor.author").get(0); itemService.removeMetadataValues(context, publication, List.of(authorToRemove)); - replaceMetadata(firstAuthor, "person", "affiliation", "name", "University"); + replaceMetadata(editor, "person", "affiliation", "name", "University"); context.restoreAuthSystemState(); @@ -357,9 +360,10 @@ public void testItemEnhancementWithForce() throws Exception { assertThat(getMetadataValues(publication, "cris.virtual.department"), hasSize(1)); assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasSize(1)); - assertThat(publication.getMetadata(), hasItem(with("cris.virtual.department", "University"))); - assertThat(publication.getMetadata(), hasItem(with("cris.virtualsource.department", firstAuthorId))); - + assertThat(getMetadataValues(publication, "cris.virtual.department"), hasItem( + with("cris.virtual.department", "University"))); + assertThat(getMetadataValues(publication, "cris.virtualsource.department"), hasItem( + with("cris.virtualsource.department", editorId))); } @Test diff --git a/dspace-server-webapp/src/main/java/org/dspace/content/enhancer/RelatedItemEnhancerUpdatePoller.java b/dspace-server-webapp/src/main/java/org/dspace/content/enhancer/RelatedItemEnhancerUpdatePoller.java new file mode 100644 index 000000000000..921edbf135e0 --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/content/enhancer/RelatedItemEnhancerUpdatePoller.java @@ -0,0 +1,63 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer; + +import java.sql.SQLException; +import java.util.UUID; + +import org.dspace.content.Item; +import org.dspace.content.enhancer.service.ItemEnhancerService; +import org.dspace.content.service.ItemService; +import org.dspace.core.Context; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +public class RelatedItemEnhancerUpdatePoller { + private static final Logger log = LoggerFactory.getLogger(RelatedItemEnhancerUpdatePoller.class); + @Autowired + private ItemEnhancerService itemEnhancerService; + + @Autowired + private ItemService itemService; + + @Scheduled(fixedDelayString = "${related-item-enhancer-poller.delay:-1}") + public void pollItemToUpdateAndProcess() { + try { + Context context = new Context(); + context.turnOffAuthorisationSystem(); + UUID extractedUuid; + while ((extractedUuid = itemEnhancerService.pollItemToUpdate(context)) != null) { + Item item = itemService.find(context, extractedUuid); + if (item != null) { + itemEnhancerService.enhance(context, item, true); + } + context.commit(); + } + context.restoreAuthSystemState(); + context.complete(); + } catch (SQLException e) { + log.error("Error polling items to update for metadata enrichment", e); + } + } + + public void setItemEnhancerService(ItemEnhancerService itemEnhancerService) { + this.itemEnhancerService = itemEnhancerService; + } + + public ItemEnhancerService getItemEnhancerService() { + return itemEnhancerService; + } + + public void setItemService(ItemService itemService) { + this.itemService = itemService; + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/batch/ImportBatchIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/batch/ImportBatchIT.java index 95306b49b930..0b34fc4025e8 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/batch/ImportBatchIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/batch/ImportBatchIT.java @@ -1157,10 +1157,10 @@ public void testDSpaceEntityTypeIsKeeped() throws SQLException { ImpRecordService.INSERT_OR_UPDATE_OPERATION, admin, publicationCollection); createImpMetadatavalue(context, impRecord, MetadataSchemaEnum.DC.getName(), "title", - null, null, "New Test publication", 0); + null, null, "New Test publication", null); createImpMetadatavalue(context, impRecord, MetadataSchemaEnum.DC.getName(), "contributor", - "author", null, "John Smith", 0); + "author", null, "John Smith", null); context.restoreAuthSystemState(); diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/matcher/CustomItemMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/matcher/CustomItemMatcher.java new file mode 100644 index 000000000000..346ab43d1732 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/app/matcher/CustomItemMatcher.java @@ -0,0 +1,26 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.matcher; + +import java.util.UUID; + +import org.dspace.content.Item; +import org.mockito.ArgumentMatcher; + +public class CustomItemMatcher implements ArgumentMatcher { + private final UUID expectedUUID; + + public CustomItemMatcher(UUID expectedUUID) { + this.expectedUUID = expectedUUID; + } + + @Override + public boolean matches(Item actual) { + return actual != null && actual.getID().equals(expectedUUID); + } +} diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java b/dspace-server-webapp/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java deleted file mode 100644 index 20295859c6fc..000000000000 --- a/dspace-server-webapp/src/test/java/org/dspace/app/matcher/MetadataValueMatcher.java +++ /dev/null @@ -1,96 +0,0 @@ -/** - * The contents of this file are subject to the license and copyright - * detailed in the LICENSE and NOTICE files at the root of the source - * tree and available online at - * - * http://www.dspace.org/license/ - */ -package org.dspace.app.matcher; - -import java.util.Objects; - -import org.dspace.content.MetadataValue; -import org.hamcrest.Description; -import org.hamcrest.TypeSafeMatcher; - -/** - * Implementation of {@link org.hamcrest.Matcher} to match a MetadataValue by - * all its attributes. - * - * @author Luca Giamminonni (luca.giamminonni at 4science.it) - * - */ -public class MetadataValueMatcher extends TypeSafeMatcher { - - private String field; - - private String value; - - private String language; - - private String authority; - - private Integer place; - - private Integer confidence; - - private MetadataValueMatcher(String field, String value, String language, String authority, Integer place, - Integer confidence) { - - this.field = field; - this.value = value; - this.language = language; - this.authority = authority; - this.place = place; - this.confidence = confidence; - - } - - @Override - public void describeTo(Description description) { - description.appendText("MetadataValue with the following attributes [field=" + field + ", value=" - + value + ", language=" + language + ", authority=" + authority + ", place=" + place + ", confidence=" - + confidence + "]"); - } - - @Override - protected void describeMismatchSafely(MetadataValue item, Description mismatchDescription) { - mismatchDescription.appendText("was ") - .appendValue("MetadataValue [metadataField=").appendValue(item.getMetadataField().toString('.')) - .appendValue(", value=").appendValue(item.getValue()).appendValue(", language=").appendValue(language) - .appendValue(", place=").appendValue(item.getPlace()).appendValue(", authority=") - .appendValue(item.getAuthority()).appendValue(", confidence=").appendValue(item.getConfidence() + "]"); - } - - @Override - protected boolean matchesSafely(MetadataValue metadataValue) { - return Objects.equals(metadataValue.getValue(), value) && - Objects.equals(metadataValue.getMetadataField().toString('.'), field) && - Objects.equals(metadataValue.getLanguage(), language) && - Objects.equals(metadataValue.getAuthority(), authority) && - Objects.equals(metadataValue.getPlace(), place) && - Objects.equals(metadataValue.getConfidence(), confidence); - } - - public static MetadataValueMatcher with(String field, String value, String language, - String authority, Integer place, Integer confidence) { - return new MetadataValueMatcher(field, value, language, authority, place, confidence); - } - - public static MetadataValueMatcher with(String field, String value) { - return with(field, value, null, null, 0, -1); - } - - public static MetadataValueMatcher with(String field, String value, String authority, int place, int confidence) { - return with(field, value, null, authority, place, confidence); - } - - public static MetadataValueMatcher with(String field, String value, String authority, int confidence) { - return with(field, value, null, authority, 0, confidence); - } - - public static MetadataValueMatcher with(String field, String value, int place) { - return with(field, value, null, null, place, -1); - } - -} \ No newline at end of file diff --git a/dspace-server-webapp/src/test/java/org/dspace/content/enhancer/RelatedItemEnhancerPollerIT.java b/dspace-server-webapp/src/test/java/org/dspace/content/enhancer/RelatedItemEnhancerPollerIT.java new file mode 100644 index 000000000000..62aca5e22945 --- /dev/null +++ b/dspace-server-webapp/src/test/java/org/dspace/content/enhancer/RelatedItemEnhancerPollerIT.java @@ -0,0 +1,334 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.content.enhancer; + +import static org.dspace.app.matcher.MetadataValueMatcher.with; +import static org.dspace.app.matcher.MetadataValueMatcher.withNoPlace; +import static org.dspace.core.CrisConstants.PLACEHOLDER_PARENT_METADATA_VALUE; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; + +import java.sql.SQLException; +import java.util.List; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.app.matcher.CustomItemMatcher; +import org.dspace.authorize.AuthorizeException; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Collection; +import org.dspace.content.Item; +import org.dspace.content.MetadataValue; +import org.dspace.content.enhancer.service.ItemEnhancerService; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.ItemService; +import org.dspace.core.ReloadableEntity; +import org.dspace.services.ConfigurationService; +import org.dspace.utils.DSpace; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class RelatedItemEnhancerPollerIT extends AbstractIntegrationTestWithDatabase { + + private ItemService itemService; + private ItemEnhancerService itemEnhancerService; + private ItemEnhancerService spyItemEnhancerService; + private RelatedItemEnhancerUpdatePoller poller = new RelatedItemEnhancerUpdatePoller(); + private Collection collection; + + @Before + public void setup() throws InterruptedException { + final DSpace dspace = new DSpace(); + ConfigurationService configurationService = dspace.getConfigurationService(); + configurationService.setProperty("item.enable-virtual-metadata", false); + itemService = ContentServiceFactory.getInstance().getItemService(); + itemEnhancerService = dspace.getSingletonService(ItemEnhancerService.class); + spyItemEnhancerService = spy(itemEnhancerService); + poller.setItemEnhancerService(spyItemEnhancerService); + poller.setItemService(itemService); + // cleanup the queue from any items left behind by other tests + poller.pollItemToUpdateAndProcess(); + context.turnOffAuthorisationSystem(); + parentCommunity = CommunityBuilder.createCommunity(context) + .withName("Parent Community") + .build(); + + collection = CollectionBuilder.createCollection(context, parentCommunity) + .withName("Collection") + .build(); + context.restoreAuthSystemState(); + } + + @Test + public void testUpdateRelatedItemAreProcessed() throws Exception { + + context.turnOffAuthorisationSystem(); + + Item person = ItemBuilder.createItem(context, collection) + .withTitle("Walter White") + .withPersonMainAffiliation("4Science") + .build(); + + String personId = person.getID().toString(); + + Item person2 = ItemBuilder.createItem(context, collection) + .withTitle("John Red") + .build(); + + String person2Id = person2.getID().toString(); + + Item person3 = ItemBuilder.createItem(context, collection) + .withTitle("Marc Green") + .withOrcidIdentifier("orcid-person3") + .withPersonMainAffiliation("Affiliation 1") + .withPersonMainAffiliation("Affiliation 2") + .build(); + + String person3Id = person3.getID().toString(); + + Item publication = ItemBuilder.createItem(context, collection) + .withTitle("Test publication") + .withEntityType("Publication") + .withAuthor("Walter White", personId) + .build(); + + Item publication2 = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 2") + .withEntityType("Publication") + .withSubject("test") + .withAuthor("Walter White", personId) + .withAuthor("John Red", person2Id) + .build(); + + Item publication3 = ItemBuilder.createItem(context, collection) + .withTitle("Test publication 3") + .withEntityType("Publication") + .withAuthor("John Red", person2Id) + .withAuthor("Marc Green", person3Id) + .build(); + + context.restoreAuthSystemState(); + publication = context.reloadEntity(publication); + publication2 = context.reloadEntity(publication2); + publication3 = context.reloadEntity(publication3); + + List metadataValues = publication.getMetadata(); + assertThat(metadataValues, hasSize(11)); + assertThat(metadataValues, hasItem(with("cris.virtual.department", "4Science"))); + assertThat(metadataValues, hasItem(with("cris.virtualsource.department", personId))); + assertThat(metadataValues, hasItem(with("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(metadataValues, hasItem(with("cris.virtualsource.orcid", personId))); + List metadataValues2 = publication2.getMetadata(); + assertThat(metadataValues2, hasSize(17)); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", "4Science"), + withNoPlace("cris.virtual.department", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", person2Id))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtual.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtualsource.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.orcid", personId), + withNoPlace("cris.virtualsource.orcid", person2Id))); + List metadataValues3 = publication3.getMetadata(); + assertThat(metadataValues3, hasSize(18)); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.department", "Affiliation 1"), + withNoPlace("cris.virtual.department", "Affiliation 2"))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", person2Id), + withNoPlace("cris.virtualsource.department", person3Id), + withNoPlace("cris.virtualsource.department", person3Id))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtual.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.orcid", "orcid-person3"))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtualsource.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.orcid", person2Id), + withNoPlace("cris.virtualsource.orcid", person3Id))); + + context.turnOffAuthorisationSystem(); + itemService.addMetadata(context, person, "person", "identifier", "orcid", null, "1234-5678-9101"); + itemService.addMetadata(context, person, "person", "affiliation", "name", null, "Company"); + itemService.update(context, person); + context.restoreAuthSystemState(); + person = commitAndReload(person); + Mockito.reset(spyItemEnhancerService); + poller.pollItemToUpdateAndProcess(); + verify(spyItemEnhancerService).enhance(any(), argThat(new CustomItemMatcher(publication.getID())), eq(true)); + verify(spyItemEnhancerService).enhance(any(), argThat(new CustomItemMatcher(publication2.getID())), eq(true)); + // 2 + 1 iteration as the last poll will return null + verify(spyItemEnhancerService, times(3)).pollItemToUpdate(any()); + verify(spyItemEnhancerService).saveAffectedItemsForUpdate(any(), eq(publication.getID())); + verify(spyItemEnhancerService).saveAffectedItemsForUpdate(any(), eq(publication2.getID())); + verifyNoMoreInteractions(spyItemEnhancerService); + person = context.reloadEntity(person); + person2 = context.reloadEntity(person2); + person3 = context.reloadEntity(person3); + publication = context.reloadEntity(publication); + publication2 = context.reloadEntity(publication2); + publication3 = context.reloadEntity(publication3); + + metadataValues = publication.getMetadata(); + assertThat(metadataValues, hasSize(13)); + assertThat(itemService.getMetadataByMetadataString(publication, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", "4Science"), + withNoPlace("cris.virtual.department", "Company"))); + assertThat(itemService.getMetadataByMetadataString(publication, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", personId))); + assertThat(metadataValues, hasItem(with("cris.virtual.orcid", "1234-5678-9101"))); + assertThat(metadataValues, hasItem(with("cris.virtualsource.orcid", personId))); + metadataValues2 = publication2.getMetadata(); + assertThat(metadataValues2, hasSize(19)); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", "4Science"), + withNoPlace("cris.virtual.department", "Company"), + withNoPlace("cris.virtual.department", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", person2Id))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtual.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtual.orcid", "1234-5678-9101"), + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtualsource.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.orcid", personId), + withNoPlace("cris.virtualsource.orcid", person2Id))); + metadataValues3 = publication3.getMetadata(); + assertThat(metadataValues3, hasSize(18)); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.department", "Affiliation 1"), + withNoPlace("cris.virtual.department", "Affiliation 2"))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", person2Id), + withNoPlace("cris.virtualsource.department", person3Id), + withNoPlace("cris.virtualsource.department", person3Id))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtual.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.orcid", "orcid-person3"))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtualsource.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.orcid", person2Id), + withNoPlace("cris.virtualsource.orcid", person3Id))); + context.turnOffAuthorisationSystem(); + itemService.clearMetadata(context, person3, "person", "identifier", "orcid", Item.ANY); + itemService.removeMetadataValues(context, person3, + List.of(itemService.getMetadataByMetadataString(person3, "person.affiliation.name").get(0))); + itemService.update(context, person3); + context.restoreAuthSystemState(); + person3 = commitAndReload(person3); + Mockito.reset(spyItemEnhancerService); + poller.pollItemToUpdateAndProcess(); + verify(spyItemEnhancerService).enhance(any(), argThat(new CustomItemMatcher(publication3.getID())), eq(true)); + // 1 + 1 iteration as the last poll will return null + verify(spyItemEnhancerService, times(2)).pollItemToUpdate(any()); + verify(spyItemEnhancerService).saveAffectedItemsForUpdate(any(), eq(publication3.getID())); + verifyNoMoreInteractions(spyItemEnhancerService); + person = context.reloadEntity(person); + person2 = context.reloadEntity(person2); + person3 = context.reloadEntity(person3); + publication = context.reloadEntity(publication); + publication2 = context.reloadEntity(publication2); + publication3 = context.reloadEntity(publication3); + + metadataValues = publication.getMetadata(); + assertThat(metadataValues, hasSize(13)); + assertThat(itemService.getMetadataByMetadataString(publication, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", "4Science"), + withNoPlace("cris.virtual.department", "Company"))); + assertThat(itemService.getMetadataByMetadataString(publication, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", personId))); + assertThat(metadataValues, hasItem(with("cris.virtual.orcid", "1234-5678-9101"))); + assertThat(metadataValues, hasItem(with("cris.virtualsource.orcid", personId))); + metadataValues2 = publication2.getMetadata(); + assertThat(metadataValues2, hasSize(19)); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", "4Science"), + withNoPlace("cris.virtual.department", "Company"), + withNoPlace("cris.virtual.department", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", personId), + withNoPlace("cris.virtualsource.department", person2Id))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtual.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtual.orcid", "1234-5678-9101"), + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication2, "cris.virtualsource.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.orcid", personId), + withNoPlace("cris.virtualsource.orcid", person2Id))); + metadataValues3 = publication3.getMetadata(); + assertThat(metadataValues3, hasSize(16)); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtual.department"), + containsInAnyOrder( + withNoPlace("cris.virtual.department", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.department", "Affiliation 2"))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtualsource.department"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.department", person2Id), + withNoPlace("cris.virtualsource.department", person3Id))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtual.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE), + withNoPlace("cris.virtual.orcid", PLACEHOLDER_PARENT_METADATA_VALUE))); + assertThat(itemService.getMetadataByMetadataString(publication3, "cris.virtualsource.orcid"), + containsInAnyOrder( + withNoPlace("cris.virtualsource.orcid", person2Id), + withNoPlace("cris.virtualsource.orcid", person3Id))); + + } + + private List getMetadataValues(Item item, String metadataField) { + return itemService.getMetadataByMetadataString(item, metadataField); + } + + @SuppressWarnings("rawtypes") + private T commitAndReload(T entity) throws SQLException, AuthorizeException { + context.commit(); + return context.reloadEntity(entity); + } + +} diff --git a/dspace/config/dspace.cfg b/dspace/config/dspace.cfg index 2e14eeec788c..f767f7822cf3 100644 --- a/dspace/config/dspace.cfg +++ b/dspace/config/dspace.cfg @@ -804,6 +804,9 @@ event.dispatcher.default.class = org.dspace.event.BasicDispatcher # Add orcidqueue here, if the integration with ORCID is configured and wish to enable the synchronization queue functionality event.dispatcher.default.consumers = versioning, discovery, eperson, dedup, crisconsumer, orcidqueue, audit, nbeventsdelete, referenceresolver, orcidwebhook, itemenhancer, customurl, reciprocal, filetypemetadataenhancer +# delay (in ms) between check for item to update due to change in related items used for metadata enrichment +# default 5 seconds +related-item-enhancer-poller.delay = 5000 # The noindex dispatcher will not create search or browse indexes (useful for batch item imports) event.dispatcher.noindex.class = org.dspace.event.BasicDispatcher diff --git a/dspace/config/modules/authority.cfg b/dspace/config/modules/authority.cfg index ee42fc52763e..b0cc1f011de2 100644 --- a/dspace/config/modules/authority.cfg +++ b/dspace/config/modules/authority.cfg @@ -275,4 +275,5 @@ authority.controlled.dc.type = true choices.plugin.dc.type = ControlledVocabularyAuthority # DSpace-CRIS stores by default the authority of controlled vocabularies -vocabulary.plugin.authority.store = true \ No newline at end of file +vocabulary.plugin.authority.store = true +authority.controlled.cris.virtual.department = true diff --git a/dspace/config/spring/api/core-dao-services.xml b/dspace/config/spring/api/core-dao-services.xml index c1518454e832..f9d8513cdb3a 100644 --- a/dspace/config/spring/api/core-dao-services.xml +++ b/dspace/config/spring/api/core-dao-services.xml @@ -87,5 +87,6 @@ + diff --git a/dspace/config/spring/api/crosswalks.xml b/dspace/config/spring/api/crosswalks.xml index 9184a56482da..8d7bb2ae9109 100644 --- a/dspace/config/spring/api/crosswalks.xml +++ b/dspace/config/spring/api/crosswalks.xml @@ -554,7 +554,9 @@ - + + diff --git a/dspace/config/spring/api/metadata-enhancers.xml b/dspace/config/spring/api/metadata-enhancers.xml index eb87c3202691..9cbae2fcb24b 100644 --- a/dspace/config/spring/api/metadata-enhancers.xml +++ b/dspace/config/spring/api/metadata-enhancers.xml @@ -23,51 +23,43 @@ + + + + Publication + Product + Patent + + - + dc.contributor.author dc.contributor.editor - - - - - - - + - dc.contributor.author - dc.contributor.editor + person.affiliation.name - - + dc.contributor.author dc.contributor.editor - - - - - - - + - dc.contributor.author - dc.contributor.editor + person.identifier.orcid -