Merged dspace-cris-2023_02_x into DSC-1586

bitbucket-pipelines.yml

@@ -46,9 +46,9 @@ definitions:
         script:
           - export MAVEN_OPTS="-Xmx4096M"
           - ./dspace-api/src/test/data/dspaceFolder/bin/install_grobid.sh
-          - mvn clean install license:check -DskipUnitTests=true -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -P !assembly -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M
+          - mvn clean install -DskipUnitTests=true -Pdspace-rest -DskipITs=true -DskipIntegrationTests=true -P !assembly -B -V
+          - mvn verify -Dmaven.test.skip=false -pl dspace-iiif,dspace-oai,dspace-rdf,dspace-rest,dspace-server-webapp,dspace-services,dspace-sword,dspace-swordv2 -DskipUnitTests=true -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -P !assembly -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M
         artifacts:
-          - dspace-api/target/surefire-reports/*-output.txt
           - dspace-iiif/target/surefire-reports/*-output.txt
           - dspace-oai/target/surefire-reports/*-output.txt
           - dspace-rdf/target/surefire-reports/*-output.txt
@@ -57,7 +57,6 @@ definitions:
           - dspace-services/target/surefire-reports/*-output.txt
           - dspace-sword/target/surefire-reports/*-output.txt
           - dspace-swordv2/target/surefire-reports/*-output.txt
-          - dspace-api/target/failsafe-reports/*-output.txt
           - dspace-iiif/target/failsafe-reports/*-output.txt
           - dspace-oai/target/failsafe-reports/*-output.txt
           - dspace-rdf/target/failsafe-reports/*-output.txt
@@ -67,21 +66,46 @@ definitions:
           - dspace-sword/target/failsafe-reports/*-output.txt
           - dspace-swordv2/target/failsafe-reports/*-output.txt
 
+    - step: &integration-tests-dspace-api
+        runs-on: self.hosted
+        name: integration-tests-dspace-api
+        size: 4x
+        caches:
+          - gradle
+          - gradlewrapper
+          - maven
+        script:
+          - export MAVEN_OPTS="-Xmx4096M"
+          - mvn clean install license:check -DskipUnitTests=true -Pdspace-rest -DskipITs=true -DskipIntegrationTests=true -P !assembly -B -V
+          - mvn verify -Dmaven.test.skip=false -pl dspace-api -DskipUnitTests=true -Pdspace-rest -DskipITs=false -Pdspace-rest -DskipIntegrationTests=false -P !assembly -B -V -Dsurefire.rerunFailingTestsCount=2 -Dtest.argLine=-Xmx4096M
+        artifacts:
+          - dspace-api/target/surefire-reports/*-output.txt
+          - dspace-api/target/failsafe-reports/*-output.txt
+
 pipelines:
   branches:
     'dspace-cris-7':
       - parallel:
         - step: *unit-test-code-checks
         - step: *integration-tests
+        - step: *integration-tests-dspace-api
+    'dspace-cris-2023_02_x':
+      - parallel:
+        - step: *unit-test-code-checks
+        - step: *integration-tests
+        - step: *integration-tests-dspace-api
 
   pull-requests:
     '**':
       - parallel:
           - step: *unit-test-code-checks
           - step: *integration-tests
+          - step: *integration-tests-dspace-api
   custom:
     integration:
        - step: *integration-tests
+    integration-dspace-api:
+       - step: *integration-tests-dspace-api
     unit:
        - step: *unit-test-code-checks
 

dspace-api/src/main/java/org/dspace/content/ItemServiceImpl.java

@@ -216,28 +216,6 @@ public Thumbnail getThumbnail(Context context, Item item) throws SQLException {
         if (thumbnail != null) {
             return thumbnail;
         }
-        // If no thumbnail is retrieved by the first strategy
-        // then use the fallback strategy
-        Bitstream thumbBitstream = null;
-        List<Bundle> originalBundles = getBundles(item, "ORIGINAL");
-        Bitstream primaryBitstream = null;
-        if (CollectionUtils.isNotEmpty(originalBundles)) {
-            primaryBitstream = originalBundles.get(0).getPrimaryBitstream();
-        }
-        if (primaryBitstream == null) {
-            primaryBitstream = bitstreamService.getFirstBitstream(item, "ORIGINAL");
-        }
-        if (primaryBitstream != null) {
-            thumbBitstream = bitstreamService.getThumbnail(context, primaryBitstream);
-            if (thumbBitstream == null) {
-                thumbBitstream = bitstreamService.getFirstBitstream(item, "THUMBNAIL");
-            }
-        }
-
-        if (thumbBitstream != null) {
-            return new Thumbnail(thumbBitstream, primaryBitstream);
-        }
-
         return null;
     }
 
@@ -252,7 +230,27 @@ private Thumbnail thumbnailLayoutTabConfigurationStrategy(Context context, Item
 
         List<CrisLayoutField> thumbFields = getThumbnailFields(crisLayoutTabs);
         if (CollectionUtils.isEmpty(thumbFields)) {
-            return null;
+            // If no thumbnail is retrieved by the first strategy
+            // then use the fallback strategy
+            Bitstream thumbBitstream = null;
+            List<Bundle> originalBundles = getBundles(item, "ORIGINAL");
+            Bitstream primaryBitstream = null;
+            if (CollectionUtils.isNotEmpty(originalBundles)) {
+                primaryBitstream = originalBundles.get(0).getPrimaryBitstream();
+            }
+            if (primaryBitstream == null) {
+                primaryBitstream = bitstreamService.getFirstBitstream(item, "ORIGINAL");
+            }
+            if (primaryBitstream != null) {
+                thumbBitstream = bitstreamService.getThumbnail(context, primaryBitstream);
+                if (thumbBitstream == null) {
+                    thumbBitstream = bitstreamService.getFirstBitstream(item, "THUMBNAIL");
+                }
+            }
+
+            if (thumbBitstream != null) {
+                return new Thumbnail(thumbBitstream, primaryBitstream);
+            }
         }
         return retrieveThumbnailFromFields(context, item, thumbFields);
     }
@@ -312,9 +310,13 @@ private Thumbnail retrieveThumbnail(Context context, Item item, String bundle,
         if (CollectionUtils.isNotEmpty(bundles)) {
             Optional<Bitstream> primaryBitstream = bundles.get(0).getBitstreams().stream().filter(bitstream -> {
                 return bitstream.getMetadata().stream().anyMatch(metadataValue -> {
-                    return metadataValue.getMetadataField().getID() == metadataField.getID()
-                        && metadataValue.getValue() != null
-                        && metadataValue.getValue().equalsIgnoreCase(value);
+                    if (metadataField != null) {
+                        return metadataValue.getMetadataField().getID() == metadataField.getID()
+                            && metadataValue.getValue() != null
+                            && metadataValue.getValue().equalsIgnoreCase(value);
+                    } else {
+                        return true;
+                    }
                 });
             }).findFirst();
             if (primaryBitstream.isEmpty()) {

dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java

@@ -31,6 +31,7 @@
  *
  */
 public class ItemEnhancerScript extends DSpaceRunnable<ItemEnhancerScriptConfiguration<ItemEnhancerScript>> {
+    private final int PAGE_SIZE = 20;
 
     private ItemService itemService;
 
@@ -57,7 +58,7 @@ public void internalRun() throws Exception {
 
         context.turnOffAuthorisationSystem();
         try {
-            enhanceItems();
+            enhanceItems(context);
             context.complete();
             handler.logInfo("Enhancement completed with success");
         } catch (Exception e) {
@@ -68,20 +69,28 @@ public void internalRun() throws Exception {
         }
     }
 
-    private void enhanceItems() {
-        findItemsToEnhance().forEachRemaining(this::enhanceItem);
+    private void enhanceItems(Context context) {
+        try {
+            int total = itemService.countArchivedItems(context);
+            for (int offset = 0; offset < total; offset += PAGE_SIZE) {
+                findItemsToEnhance(offset).forEachRemaining(this::enhanceItem);
+                context.commit();
+                context.clear();
+            }
+        } catch (SQLException e) {
+            throw new SQLRuntimeException(e);
+        }
     }
 
-    private Iterator<Item> findItemsToEnhance() {
+    private Iterator<Item> findItemsToEnhance(int offset) {
         try {
-            return itemService.findAll(context);
+            return itemService.findAll(context, PAGE_SIZE, offset);
         } catch (SQLException e) {
             throw new SQLRuntimeException(e);
         }
     }
 
     private void enhanceItem(Item item) {
-
         itemEnhancerService.enhance(context, item, force);
         uncacheItem(item);
     }

dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java

@@ -24,7 +24,15 @@ public interface ItemEnhancerService {
 
     /**
      * Enhances the given item with all the item enhancers defined adding virtual
-     * metadata fields on it.
+     * metadata fields on it. ItemEnhancer will use the information stored in the
+     * source metadata to decide if virtual metadata must be calculated. This could
+     * lead to stale information if the given item is linked to the same related items
+     * than before but in the mean time the related items have been changed in a way
+     * that could affect the generated virtual metadata (for example a publication
+     * listing 3 authors assuming that we are flatting on the publication the information
+     * about the author current affiliation would not update the virtual affiliation
+     * if this method is invoked on the item without touching the author list - in this
+     * scenario you need to use the {@link #forceEnhancement(Context, Item)} method
      *
      * @param context the DSpace Context
      * @param item    the item to enhance

dspace-api/src/main/java/org/dspace/core/Context.java

@@ -23,6 +23,7 @@
 import org.apache.logging.log4j.Logger;
 import org.dspace.authorize.ResourcePolicy;
 import org.dspace.content.DSpaceObject;
+import org.dspace.core.exception.SQLRuntimeException;
 import org.dspace.eperson.EPerson;
 import org.dspace.eperson.Group;
 import org.dspace.eperson.factory.EPersonServiceFactory;
@@ -33,6 +34,7 @@
 import org.dspace.storage.rdbms.DatabaseConfigVO;
 import org.dspace.storage.rdbms.DatabaseUtils;
 import org.dspace.utils.DSpace;
+import org.hibernate.Session;
 import org.springframework.util.CollectionUtils;
 
 /**
@@ -451,6 +453,14 @@ public void commit() throws SQLException {
         }
     }
 
+    public void clear() {
+        try {
+            ((Session) dbConnection.getSession()).clear();
+            reloadContextBoundEntities();
+        } catch (SQLException e) {
+            throw new SQLRuntimeException(e);
+        }
+    }
 
     /**
      * Dispatch any events (cached in current Context) to configured EventListeners (consumers)

dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java

@@ -20,6 +20,7 @@
 import org.dspace.content.DSpaceObject;
 import org.dspace.content.Item;
 import org.dspace.handle.Handle;
+import org.dspace.identifier.DOI;
 import org.dspace.storage.rdbms.DatabaseConfigVO;
 import org.hibernate.FlushMode;
 import org.hibernate.Hibernate;
@@ -280,6 +281,11 @@ public <E extends ReloadableEntity> void uncacheEntity(E entity) throws SQLExcep
                 }
             }
 
+            if (entity instanceof DOI) {
+                DOI doi = (DOI) entity;
+                uncacheEntity(doi.getDSpaceObject());
+            }
+
             // ITEM
             if (entity instanceof Item) {
                 Item item = (Item) entity;