From d80f49e0238981c22703abbb1423aa2896eca694 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Tue, 23 Jul 2024 18:28:43 +0200 Subject: [PATCH 1/4] 116609: Improve running process observability - keep temp process log files in [dspace]/log/processes/ instead of temp dir - reformat file names of process logs - ensure that running and scheduled processes are cleaned up during startup --- .../dspace/scripts/ProcessServiceImpl.java | 56 +++++++++++++++---- .../app/rest/ProcessRestRepositoryIT.java | 8 +-- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java index 2e14aeaa36c0..63751d87e92a 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java @@ -45,14 +45,15 @@ import org.dspace.core.LogHelper; import org.dspace.eperson.EPerson; import org.dspace.eperson.Group; -import org.dspace.eperson.service.EPersonService; import org.dspace.scripts.service.ProcessService; +import org.dspace.services.ConfigurationService; +import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Autowired; /** * The implementation for the {@link ProcessService} class */ -public class ProcessServiceImpl implements ProcessService { +public class ProcessServiceImpl implements ProcessService, InitializingBean { private static final Logger log = org.apache.logging.log4j.LogManager.getLogger(ProcessService.class); @@ -72,7 +73,26 @@ public class ProcessServiceImpl implements ProcessService { private MetadataFieldService metadataFieldService; @Autowired - private EPersonService ePersonService; + private ConfigurationService configurationService; + + @Override + public void afterPropertiesSet() throws Exception { + Context context = new Context(); + + // Processes that were running or scheduled when tomcat crashed, should be cleaned up during startup. + List processesToBeFailed = findByStatusAndCreationTimeOlderThan( + context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), new Date()); + for (Process process : processesToBeFailed) { + context.setCurrentUser(process.getEPerson()); + // Fail the process. + log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID()); + fail(context, process); + // But still attach its log to the process. + createLogBitstream(context, process); + } + + context.complete(); + } @Override public Process create(Context context, EPerson ePerson, String scriptName, @@ -286,8 +306,8 @@ public int countSearch(Context context, ProcessQueryParameterContainer processQu @Override public void appendLog(int processId, String scriptName, String output, ProcessLogLevel processLogLevel) throws IOException { - File tmpDir = FileUtils.getTempDirectory(); - File tempFile = new File(tmpDir, scriptName + processId + ".log"); + File logsDir = getLogsDirectory(); + File tempFile = new File(logsDir, processId + "-" + scriptName + ".log"); FileWriter out = new FileWriter(tempFile, true); try { try (BufferedWriter writer = new BufferedWriter(out)) { @@ -302,12 +322,15 @@ public void appendLog(int processId, String scriptName, String output, ProcessLo @Override public void createLogBitstream(Context context, Process process) throws IOException, SQLException, AuthorizeException { - File tmpDir = FileUtils.getTempDirectory(); - File tempFile = new File(tmpDir, process.getName() + process.getID() + ".log"); - FileInputStream inputStream = FileUtils.openInputStream(tempFile); - appendFile(context, process, inputStream, Process.OUTPUT_TYPE, process.getName() + process.getID() + ".log"); - inputStream.close(); - tempFile.delete(); + File logsDir = getLogsDirectory(); + File tempFile = new File(logsDir, process.getID() + "-" + process.getName() + ".log"); + if (tempFile.exists()) { + FileInputStream inputStream = FileUtils.openInputStream(tempFile); + appendFile(context, process, inputStream, Process.OUTPUT_TYPE, + process.getID() + "-" + process.getName() + ".log"); + inputStream.close(); + tempFile.delete(); + } } @Override @@ -336,4 +359,15 @@ private String formatLogLine(int processId, String scriptName, String output, Pr return sb.toString(); } + private File getLogsDirectory() { + String pathStr = configurationService.getProperty("dspace.dir") + + File.separator + "log" + File.separator + "processes"; + File logsDir = new File(pathStr); + if (!logsDir.exists()) { + if (!logsDir.mkdirs()) { + throw new RuntimeException("Couldn't create [dspace.dir]/log/processes/ directory."); + } + } + return logsDir; + } } diff --git a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java index 670d8e2f35b0..6c018df6d070 100644 --- a/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java +++ b/dspace-server-webapp/src/test/java/org/dspace/app/rest/ProcessRestRepositoryIT.java @@ -856,10 +856,10 @@ public void getProcessOutput() throws Exception { getClient(token).perform(get("/api/system/processes/" + process1.getID() + "/output")) .andExpect(status().isOk()) .andExpect(jsonPath("$.name", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.type", is("bitstream"))) .andExpect(jsonPath("$.metadata['dc.title'][0].value", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.metadata['dspace.process.filetype'][0].value", is("script_output"))); @@ -869,10 +869,10 @@ public void getProcessOutput() throws Exception { .perform(get("/api/system/processes/" + process1.getID() + "/output")) .andExpect(status().isOk()) .andExpect(jsonPath("$.name", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.type", is("bitstream"))) .andExpect(jsonPath("$.metadata['dc.title'][0].value", - is(process1.getName() + process1.getID() + ".log"))) + is(process1.getID() + "-" + process1.getName() + ".log"))) .andExpect(jsonPath("$.metadata['dspace.process.filetype'][0].value", is("script_output"))); From 156ad471b575dfc1c2b643ec4b2f677d6f89a314 Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Thu, 25 Jul 2024 09:33:50 +0200 Subject: [PATCH 2/4] 116609: Add tomcat shutdown line to process log --- .../src/main/java/org/dspace/scripts/ProcessServiceImpl.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java index 63751d87e92a..2f34fac65833 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java @@ -88,6 +88,9 @@ public void afterPropertiesSet() throws Exception { log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID()); fail(context, process); // But still attach its log to the process. + appendLog(process.getID(), process.getName(), + "Process did not complete before tomcat shutdown.", + ProcessLogLevel.ERROR); createLogBitstream(context, process); } From bdf7069cb752782a4f8df62885c9ae6116be9dde Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Thu, 25 Jul 2024 14:13:16 +0200 Subject: [PATCH 3/4] 116687: Never handle exception with null message --- .../rest/scripts/handler/impl/RestDSpaceRunnableHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java index 596ab4429093..ee67baa8ab38 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/scripts/handler/impl/RestDSpaceRunnableHandler.java @@ -130,7 +130,7 @@ public void handleCompletion() { @Override public void handleException(Exception e) { - handleException(null, e); + handleException(e.getMessage(), e); } @Override From 070fe689d70d8dbfcde63899d377f2524f38d6db Mon Sep 17 00:00:00 2001 From: Nona Luypaert Date: Mon, 29 Jul 2024 13:48:32 +0200 Subject: [PATCH 4/4] 116609: Add try catch to init method in ProcessServiceImpl --- .../dspace/scripts/ProcessServiceImpl.java | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java index c34705a64d5d..ab5147221cfc 100644 --- a/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java +++ b/dspace-api/src/main/java/org/dspace/scripts/ProcessServiceImpl.java @@ -77,24 +77,29 @@ public class ProcessServiceImpl implements ProcessService, InitializingBean { @Override public void afterPropertiesSet() throws Exception { - Context context = new Context(); - - // Processes that were running or scheduled when tomcat crashed, should be cleaned up during startup. - List processesToBeFailed = findByStatusAndCreationTimeOlderThan( - context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), new Date()); - for (Process process : processesToBeFailed) { - context.setCurrentUser(process.getEPerson()); - // Fail the process. - log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", process.getID()); - fail(context, process); - // But still attach its log to the process. - appendLog(process.getID(), process.getName(), - "Process did not complete before tomcat shutdown.", - ProcessLogLevel.ERROR); - createLogBitstream(context, process); - } + try { + Context context = new Context(); + + // Processes that were running or scheduled when tomcat crashed, should be cleaned up during startup. + List processesToBeFailed = findByStatusAndCreationTimeOlderThan( + context, List.of(ProcessStatus.RUNNING, ProcessStatus.SCHEDULED), new Date()); + for (Process process : processesToBeFailed) { + context.setCurrentUser(process.getEPerson()); + // Fail the process. + log.info("Process with ID {} did not complete before tomcat shutdown, failing it now.", + process.getID()); + fail(context, process); + // But still attach its log to the process. + appendLog(process.getID(), process.getName(), + "Process did not complete before tomcat shutdown.", + ProcessLogLevel.ERROR); + createLogBitstream(context, process); + } - context.complete(); + context.complete(); + } catch (Exception e) { + log.error("Unable to clean up Processes: ", e); + } } @Override