From b815bed1120fb066274ba353c9548cc0d0c55b0a Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Tue, 9 Jan 2024 19:39:39 +0100 Subject: [PATCH 1/3] CST-13236 Improve performance of ItemEnhancer working in an incremental way at the database level --- .../content/enhancer/script/ItemEnhancerScript.java | 13 ++++++------- .../enhancer/service/ItemEnhancerService.java | 13 +++++++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java index 2c4d1f203468..a98e423cd26e 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java @@ -57,7 +57,7 @@ public void internalRun() throws Exception { context.turnOffAuthorisationSystem(); try { - enhanceItems(); + enhanceItems(context); context.complete(); handler.logInfo("Enhancement completed with success"); } catch (Exception e) { @@ -68,7 +68,7 @@ public void internalRun() throws Exception { } } - private void enhanceItems() { + private void enhanceItems(Context context) { findItemsToEnhance().forEachRemaining(this::enhanceItem); } @@ -87,14 +87,13 @@ private void enhanceItem(Item item) { } else { itemEnhancerService.enhance(context, item); } - - uncacheItem(item); - + storeChangesAndFreeResources(context); } - private void uncacheItem(Item item) { + private void storeChangesAndFreeResources(Context context) { try { - context.uncacheEntity(item); + context.commit(); + context.clear(); } catch (SQLException e) { throw new SQLRuntimeException(e); } diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java b/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java index 5b3b419bfa8f..62349b06bdf2 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/service/ItemEnhancerService.java @@ -20,7 +20,15 @@ public interface ItemEnhancerService { /** * Enhances the given item with all the item enhancers defined adding virtual - * metadata fields on it. + * metadata fields on it. ItemEnhancer will use the information stored in the + * source metadata to decide if virtual metadata must be calculated. This could + * lead to stale information if the given item is linked to the same related items + * than before but in the mean time the related items have been changed in a way + * that could affect the generated virtual metadata (for example a publication + * listing 3 authors assuming that we are flatting on the publication the information + * about the author current affiliation would not update the virtual affiliation + * if this method is invoked on the item without touching the author list - in this + * scenario you need to use the {@link #forceEnhancement(Context, Item)} method * * @param context the DSpace Context * @param item the item to enhance @@ -29,7 +37,8 @@ public interface ItemEnhancerService { /** * Remove all the already calculated virtual metadata fields from the given item - * and perform a new enhancement. + * and perform a new enhancement. This remove the risk of stale data related to + * unforeseen update on the related items * * @param context the DSpace Context * @param item the item to enhance From 5da479524b4375471b2e02d1613b641171c49617 Mon Sep 17 00:00:00 2001 From: Andrea Bollini Date: Wed, 10 Jan 2024 22:27:25 +0100 Subject: [PATCH 2/3] CST-13236 iterate over all the items in a paginated way committing after each page --- .../enhancer/script/ItemEnhancerScript.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java index a98e423cd26e..d67063210c6e 100644 --- a/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java +++ b/dspace-api/src/main/java/org/dspace/content/enhancer/script/ItemEnhancerScript.java @@ -31,6 +31,7 @@ * */ public class ItemEnhancerScript extends DSpaceRunnable> { + private final int PAGE_SIZE = 20; private ItemService itemService; @@ -69,34 +70,32 @@ public void internalRun() throws Exception { } private void enhanceItems(Context context) { - findItemsToEnhance().forEachRemaining(this::enhanceItem); + try { + int total = itemService.countArchivedItems(context); + for (int offset = 0; offset < total; offset += PAGE_SIZE) { + findItemsToEnhance(offset).forEachRemaining(this::enhanceItem); + } + context.commit(); + context.clear(); + } catch (SQLException e) { + throw new SQLRuntimeException(e); + } } - private Iterator findItemsToEnhance() { + private Iterator findItemsToEnhance(int offset) { try { - return itemService.findAll(context); + return itemService.findAll(context, PAGE_SIZE, offset); } catch (SQLException e) { throw new SQLRuntimeException(e); } } private void enhanceItem(Item item) { - if (force) { itemEnhancerService.forceEnhancement(context, item); } else { itemEnhancerService.enhance(context, item); } - storeChangesAndFreeResources(context); - } - - private void storeChangesAndFreeResources(Context context) { - try { - context.commit(); - context.clear(); - } catch (SQLException e) { - throw new SQLRuntimeException(e); - } } private void assignCurrentUserInContext() throws SQLException { From a847dd14ac31292a1d376c7b82162c4138cad96a Mon Sep 17 00:00:00 2001 From: Nikita Krivonosov Date: Tue, 30 Jan 2024 20:19:30 +0100 Subject: [PATCH 3/3] [DSC-1473] - ItemEnhancer: improve the performance of the script --- dspace-api/src/main/java/org/dspace/core/Context.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index 4d3079240c3d..ad82ee83a841 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.Logger; import org.dspace.authorize.ResourcePolicy; import org.dspace.content.DSpaceObject; +import org.dspace.core.exception.SQLRuntimeException; import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; import org.dspace.eperson.factory.EPersonServiceFactory; @@ -33,6 +34,7 @@ import org.dspace.storage.rdbms.DatabaseConfigVO; import org.dspace.storage.rdbms.DatabaseUtils; import org.dspace.utils.DSpace; +import org.hibernate.Session; import org.springframework.util.CollectionUtils; /** @@ -451,6 +453,14 @@ public void commit() throws SQLException { } } + public void clear() { + try { + ((Session) dbConnection.getSession()).clear(); + reloadContextBoundEntities(); + } catch (SQLException e) { + throw new SQLRuntimeException(e); + } + } /** * Dispatch any events (cached in current Context) to configured EventListeners (consumers)