From cd24af5aba460d5bb0c905395ffadd5805dc5760 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Thu, 12 Dec 2024 08:34:48 +0000 Subject: [PATCH 01/10] added changes for FACT-2041 --- .../reform/sendletter/launchdarkly/Flags.java | 1 + .../tasks/DeleteOldLettersTask.java | 91 +++++++++++++++++++ .../tasks/DeleteOldLettersTaskTest.java | 74 +++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java create mode 100644 src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/launchdarkly/Flags.java b/src/main/java/uk/gov/hmcts/reform/sendletter/launchdarkly/Flags.java index d3e2c4221..73105162c 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/launchdarkly/Flags.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/launchdarkly/Flags.java @@ -6,4 +6,5 @@ private Flags() { } public static final String SEND_LETTER_SERVICE_TEST = "send-letter-service-test"; + public static final String SEND_LETTER_SERVICE_DELETE_LETTERS_CRON = "send-letter-service-delete-letters-cron"; } diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java new file mode 100644 index 000000000..4fd8c745c --- /dev/null +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -0,0 +1,91 @@ +package uk.gov.hmcts.reform.sendletter.tasks; + +import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import uk.gov.hmcts.reform.sendletter.launchdarkly.LaunchDarklyClient; + +import static uk.gov.hmcts.reform.sendletter.launchdarkly.Flags.SEND_LETTER_SERVICE_DELETE_LETTERS_CRON; +import static uk.gov.hmcts.reform.sendletter.util.TimeZones.EUROPE_LONDON; + +/** + * Deletes old letters from the database based on a hardcoded SQL query. + * This task runs in all environments. + */ +@Component +public class DeleteOldLettersTask { + + private static final Logger logger = LoggerFactory.getLogger(DeleteOldLettersTask.class); + + private static final String TASK_NAME = "DeleteOldLetters"; + + private final JdbcTemplate jdbcTemplate; + + private final LaunchDarklyClient launchDarklyClient; + + + private static final String DELETE_QUERY = """ + WITH letters_to_delete AS ( + SELECT id -- selecting indexed id column is all we need. also is faster then doing 'select *' + FROM letters + WHERE created_at < ( + CASE + WHEN service = 'civil_general_applications' THEN NOW() - INTERVAL '6 YEARS' + WHEN service = 'civil_service' THEN NOW() - INTERVAL '6 YEARS' + WHEN service = 'cmc_claim_store' THEN NOW() - INTERVAL '2 YEARS' + WHEN service = 'divorce_frontend' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'finrem_case_orchestration' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'finrem_document_generator' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'fpl_case_service' THEN NOW() - INTERVAL '2 YEARS' + WHEN service = 'nfdiv_case_api' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'prl_cos_api' THEN NOW() - INTERVAL '18 YEARS' + WHEN service = 'probate_backend' THEN NOW() - INTERVAL '1 YEAR' + WHEN service = 'send_letter_tests' THEN NOW() - INTERVAL '2 YEARS' + WHEN service = 'sscs' THEN NOW() - INTERVAL '3 MONTHS' + -- no default case + END + ) + -- For below don't delete old unprocessed / things that need investigating at some point + AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') + ORDER BY created_at ASC -- Prioritize oldest rows first + LIMIT 30000 -- Limit so it doesn't delete too many at once but needs to be high enough to make a difference + ) + DELETE FROM letters + USING letters_to_delete + WHERE letters.id = letters_to_delete.id; + """; + + /** + * Constructor for the DeleteOldLettersTask. + * @param jdbcTemplate The JDBC template for running SQL queries. + */ + public DeleteOldLettersTask(JdbcTemplate jdbcTemplate, LaunchDarklyClient launchDarklyClient) { + this.jdbcTemplate = jdbcTemplate; + this.launchDarklyClient = launchDarklyClient; + } + + /** + * Run the task to delete old letters from the database. + */ + @SchedulerLock(name = TASK_NAME) + @Scheduled(cron = "0 0 17 ? * SAT", zone = EUROPE_LONDON) // Every Saturday at 5 PM + //@Scheduled(cron = "0 */2 * * * ?", zone = EUROPE_LONDON) // Every 2 minutes for testing + public void run() { + logger.info("Starting {} task", TASK_NAME); + if(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)){ + logger.info("Flag enabled. Task {} running", TASK_NAME); + try { + int rowsDeleted = jdbcTemplate.update(DELETE_QUERY); + logger.info("{} task completed. {} rows deleted", TASK_NAME, rowsDeleted); + } catch (Exception e) { + logger.error("Error occurred during {} task", TASK_NAME, e); + } + } else { + logger.info("Flag disabled. Task {} did not run", TASK_NAME); + } + + } +} diff --git a/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java b/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java new file mode 100644 index 000000000..d3013c195 --- /dev/null +++ b/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java @@ -0,0 +1,74 @@ +package uk.gov.hmcts.reform.sendletter.tasks; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.jdbc.core.JdbcTemplate; +import uk.gov.hmcts.reform.sendletter.launchdarkly.LaunchDarklyClient; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static uk.gov.hmcts.reform.sendletter.launchdarkly.Flags.SEND_LETTER_SERVICE_DELETE_LETTERS_CRON; + +class DeleteOldLettersTaskTest { + + @Mock + private JdbcTemplate jdbcTemplate; + + @Mock + private LaunchDarklyClient launchDarklyClient; + + private DeleteOldLettersTask deleteOldLettersTask; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + deleteOldLettersTask = new DeleteOldLettersTask(jdbcTemplate, launchDarklyClient); + } + + @Test + void shouldExecuteDeleteQueryWhenFeatureFlagIsEnabled() { + // Given + when(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)).thenReturn(true); + when(jdbcTemplate.update(anyString())).thenReturn(100); + + // When + deleteOldLettersTask.run(); + + // Then + verify(launchDarklyClient, times(1)).isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON); + verify(jdbcTemplate, times(1)).update(anyString()); + } + + @Test + void shouldNotExecuteDeleteQueryWhenFeatureFlagIsDisabled() { + // Given + when(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)).thenReturn(false); + + // When + deleteOldLettersTask.run(); + + // Then + verify(launchDarklyClient, times(1)).isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON); + verify(jdbcTemplate, never()).update(anyString()); + } + + @Test + void shouldHandleDatabaseErrorGracefullyWhenFeatureFlagIsEnabled() { + // Given + when(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)).thenReturn(true); + doThrow(new RuntimeException("Database error")).when(jdbcTemplate).update(anyString()); + + // When + deleteOldLettersTask.run(); + + // Then + verify(launchDarklyClient, times(1)).isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON); + verify(jdbcTemplate, times(1)).update(anyString()); + } +} From 2503dcffaba07c73c8b1a429a6af146828653841 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Thu, 12 Dec 2024 08:42:45 +0000 Subject: [PATCH 02/10] checkstylemain --- .../hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java index 4fd8c745c..f907967d8 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -56,7 +56,7 @@ AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') DELETE FROM letters USING letters_to_delete WHERE letters.id = letters_to_delete.id; - """; + """; /** * Constructor for the DeleteOldLettersTask. @@ -75,7 +75,7 @@ public DeleteOldLettersTask(JdbcTemplate jdbcTemplate, LaunchDarklyClient launch //@Scheduled(cron = "0 */2 * * * ?", zone = EUROPE_LONDON) // Every 2 minutes for testing public void run() { logger.info("Starting {} task", TASK_NAME); - if(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)){ + if (launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)) { logger.info("Flag enabled. Task {} running", TASK_NAME); try { int rowsDeleted = jdbcTemplate.update(DELETE_QUERY); From 870ee50d7368dbb518008401ad97b91249b809c3 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Thu, 12 Dec 2024 15:20:07 +0000 Subject: [PATCH 03/10] added changes to make it better --- .../tasks/DeleteOldLettersTask.java | 48 +++++++------------ ...028__Add_batch_delete_letters_function.sql | 47 ++++++++++++++++++ 2 files changed, 63 insertions(+), 32 deletions(-) create mode 100644 src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java index f907967d8..f14a9f8da 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -8,6 +8,8 @@ import org.springframework.stereotype.Component; import uk.gov.hmcts.reform.sendletter.launchdarkly.LaunchDarklyClient; +import java.util.Optional; + import static uk.gov.hmcts.reform.sendletter.launchdarkly.Flags.SEND_LETTER_SERVICE_DELETE_LETTERS_CRON; import static uk.gov.hmcts.reform.sendletter.util.TimeZones.EUROPE_LONDON; @@ -26,37 +28,9 @@ public class DeleteOldLettersTask { private final LaunchDarklyClient launchDarklyClient; + private final Integer BATCH_SIZE = 1000; - private static final String DELETE_QUERY = """ - WITH letters_to_delete AS ( - SELECT id -- selecting indexed id column is all we need. also is faster then doing 'select *' - FROM letters - WHERE created_at < ( - CASE - WHEN service = 'civil_general_applications' THEN NOW() - INTERVAL '6 YEARS' - WHEN service = 'civil_service' THEN NOW() - INTERVAL '6 YEARS' - WHEN service = 'cmc_claim_store' THEN NOW() - INTERVAL '2 YEARS' - WHEN service = 'divorce_frontend' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'finrem_case_orchestration' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'finrem_document_generator' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'fpl_case_service' THEN NOW() - INTERVAL '2 YEARS' - WHEN service = 'nfdiv_case_api' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'prl_cos_api' THEN NOW() - INTERVAL '18 YEARS' - WHEN service = 'probate_backend' THEN NOW() - INTERVAL '1 YEAR' - WHEN service = 'send_letter_tests' THEN NOW() - INTERVAL '2 YEARS' - WHEN service = 'sscs' THEN NOW() - INTERVAL '3 MONTHS' - -- no default case - END - ) - -- For below don't delete old unprocessed / things that need investigating at some point - AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') - ORDER BY created_at ASC -- Prioritize oldest rows first - LIMIT 30000 -- Limit so it doesn't delete too many at once but needs to be high enough to make a difference - ) - DELETE FROM letters - USING letters_to_delete - WHERE letters.id = letters_to_delete.id; - """; + private final String DELETE_QUERY = "SELECT batch_delete_letters(?);"; /** * Constructor for the DeleteOldLettersTask. @@ -77,9 +51,19 @@ public void run() { logger.info("Starting {} task", TASK_NAME); if (launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)) { logger.info("Flag enabled. Task {} running", TASK_NAME); + + int totalRowsDeleted = 0; + int rowsDeleted; + try { - int rowsDeleted = jdbcTemplate.update(DELETE_QUERY); - logger.info("{} task completed. {} rows deleted", TASK_NAME, rowsDeleted); + do { + rowsDeleted = Optional.ofNullable( + jdbcTemplate.queryForObject(DELETE_QUERY, new Object[]{BATCH_SIZE}, Integer.class) + ).orElse(0); + totalRowsDeleted += rowsDeleted; + logger.info("Batch deleted: {} rows, Total deleted: {} rows", rowsDeleted, totalRowsDeleted); + } while (rowsDeleted > 0); + logger.info("{} task completed. Total rows deleted: {}", TASK_NAME, totalRowsDeleted); } catch (Exception e) { logger.error("Error occurred during {} task", TASK_NAME, e); } diff --git a/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql b/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql new file mode 100644 index 000000000..1c60ccdcf --- /dev/null +++ b/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql @@ -0,0 +1,47 @@ +CREATE OR REPLACE FUNCTION batch_delete_letters(batch_size INT) +RETURNS INT AS +$$ +DECLARE +rows_deleted INT; +BEGIN + -- Adding manual locks but have to lock each manually since we using delete cascade. + -- lock released automatically at end of transaction/commit. + LOCK TABLE letters, documents, letter_events IN ACCESS EXCLUSIVE MODE; + + -- Perform deletion in a batch and get the number of rows deleted + WITH letters_to_delete AS ( + SELECT id -- Selecting indexed id column is all we need, for better performance + FROM letters + WHERE created_at < ( + CASE + WHEN service = 'civil_general_applications' THEN NOW() - INTERVAL '6 YEARS' + WHEN service = 'civil_service' THEN NOW() - INTERVAL '6 YEARS' + WHEN service = 'cmc_claim_store' THEN NOW() - INTERVAL '2 YEARS' + WHEN service = 'divorce_frontend' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'finrem_case_orchestration' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'finrem_document_generator' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'fpl_case_service' THEN NOW() - INTERVAL '2 YEARS' + WHEN service = 'nfdiv_case_api' THEN NOW() - INTERVAL '3 MONTHS' + WHEN service = 'prl_cos_api' THEN NOW() - INTERVAL '18 YEARS' + WHEN service = 'probate_backend' THEN NOW() - INTERVAL '1 YEAR' + WHEN service = 'send_letter_tests' THEN NOW() - INTERVAL '2 YEARS' + WHEN service = 'sscs' THEN NOW() - INTERVAL '3 MONTHS' + -- no default case + END + ) + -- For below don't delete old unprocessed / things that need investigating at some point + AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') + ORDER BY created_at ASC -- Prioritize oldest rows first + LIMIT batch_size -- Limit the number of rows selected in each batch + ) + DELETE FROM letters + USING letters_to_delete + WHERE letters.id = letters_to_delete.id; + + -- Get the number of rows deleted by the DELETE statement + GET DIAGNOSTICS rows_deleted = ROW_COUNT; + + -- Return the number of rows deleted + RETURN rows_deleted; +END; +$$ LANGUAGE plpgsql; From 07d4491ab1236fb023c52d8ba413e40316f74aa5 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Thu, 12 Dec 2024 15:29:21 +0000 Subject: [PATCH 04/10] checkstylemain --- .../reform/sendletter/tasks/DeleteOldLettersTask.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java index f14a9f8da..c4646b1d6 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -28,9 +28,10 @@ public class DeleteOldLettersTask { private final LaunchDarklyClient launchDarklyClient; - private final Integer BATCH_SIZE = 1000; + private final Integer batchSize = 1000; - private final String DELETE_QUERY = "SELECT batch_delete_letters(?);"; + // See V028__Add_batch_delete_letters_function.sql for sql function + private final String deleteQuery = "SELECT batch_delete_letters(?);"; /** * Constructor for the DeleteOldLettersTask. @@ -42,7 +43,7 @@ public DeleteOldLettersTask(JdbcTemplate jdbcTemplate, LaunchDarklyClient launch } /** - * Run the task to delete old letters from the database. + * Deletes old letters from the database in batches based on the batch_delete_letters query. */ @SchedulerLock(name = TASK_NAME) @Scheduled(cron = "0 0 17 ? * SAT", zone = EUROPE_LONDON) // Every Saturday at 5 PM @@ -58,7 +59,7 @@ public void run() { try { do { rowsDeleted = Optional.ofNullable( - jdbcTemplate.queryForObject(DELETE_QUERY, new Object[]{BATCH_SIZE}, Integer.class) + jdbcTemplate.queryForObject(deleteQuery, new Object[]{batchSize}, Integer.class) ).orElse(0); totalRowsDeleted += rowsDeleted; logger.info("Batch deleted: {} rows, Total deleted: {} rows", rowsDeleted, totalRowsDeleted); From 4ca0c77f75399f9d7ded28b927c8f215f12b6d78 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Thu, 12 Dec 2024 15:53:54 +0000 Subject: [PATCH 05/10] changed tests --- .../tasks/DeleteOldLettersTaskTest.java | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java b/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java index d3013c195..efd2f3c2c 100644 --- a/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java +++ b/src/test/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTaskTest.java @@ -7,8 +7,9 @@ import org.springframework.jdbc.core.JdbcTemplate; import uk.gov.hmcts.reform.sendletter.launchdarkly.LaunchDarklyClient; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doThrow; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -32,43 +33,46 @@ void setUp() { } @Test - void shouldExecuteDeleteQueryWhenFeatureFlagIsEnabled() { - // Given + void testRunWhenFeatureFlagIsEnabled() { + // Arrange: Mock LaunchDarkly feature flag to return true when(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)).thenReturn(true); - when(jdbcTemplate.update(anyString())).thenReturn(100); - // When + // Mock JdbcTemplate behavior for the deletion loop + when(jdbcTemplate.queryForObject(anyString(), any(Object[].class), eq(Integer.class))) + .thenReturn(100) + .thenReturn(50) + .thenReturn(0); // Simulate 3 batches with 100, 50 rows deleted, and then no more rows + + // Act: Call the run method deleteOldLettersTask.run(); - // Then - verify(launchDarklyClient, times(1)).isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON); - verify(jdbcTemplate, times(1)).update(anyString()); + // Assert: Check if the correct number of deletions were made + verify(jdbcTemplate, times(3)).queryForObject(anyString(), any(Object[].class), eq(Integer.class)); } @Test - void shouldNotExecuteDeleteQueryWhenFeatureFlagIsDisabled() { - // Given + void testRunWhenFeatureFlagIsDisabled() { + // Arrange: Mock LaunchDarkly feature flag to return false when(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)).thenReturn(false); - // When + // Act: Call the run method deleteOldLettersTask.run(); - // Then - verify(launchDarklyClient, times(1)).isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON); - verify(jdbcTemplate, never()).update(anyString()); + // Assert: Verify no deletion attempt is made + verify(jdbcTemplate, never()).queryForObject(anyString(), any(Object[].class), eq(Integer.class)); } @Test - void shouldHandleDatabaseErrorGracefullyWhenFeatureFlagIsEnabled() { - // Given + void testRunWhenErrorOccurs() { + // Arrange: Mock LaunchDarkly feature flag to return true when(launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)).thenReturn(true); - doThrow(new RuntimeException("Database error")).when(jdbcTemplate).update(anyString()); - // When + // Mock JdbcTemplate to throw an exception + when(jdbcTemplate.queryForObject(anyString(), any(Object[].class), eq(Integer.class))) + .thenThrow(new RuntimeException("Database error")); + + // Act: Call the run method deleteOldLettersTask.run(); - // Then - verify(launchDarklyClient, times(1)).isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON); - verify(jdbcTemplate, times(1)).update(anyString()); } } From 42c188beb3606f1ab4f73509e2ae74dc4a6bf2d4 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Mon, 16 Dec 2024 16:21:50 +0000 Subject: [PATCH 06/10] altererd to be configurable intervals --- charts/rpe-send-letter-service/values.yaml | 15 ++++ .../tasks/DeleteOldLettersTask.java | 82 +++++++++++++++-- src/main/resources/application.yaml | 17 ++++ ...028__Add_batch_delete_letters_function.sql | 89 +++++++++++-------- 4 files changed, 161 insertions(+), 42 deletions(-) diff --git a/charts/rpe-send-letter-service/values.yaml b/charts/rpe-send-letter-service/values.yaml index 9c41f2ce2..9a52d82b6 100644 --- a/charts/rpe-send-letter-service/values.yaml +++ b/charts/rpe-send-letter-service/values.yaml @@ -80,5 +80,20 @@ java: FTP_UPLOADED_LETTERS_SUMMARY_REPORT_ENABLED: "false" DUPLICATES_CUT_OFF_TIME: 1 + # delete old letters interval times + DELETE_OLD_LETTERS_BATCH_SIZE: 1000 + DELETE_OLD_LETTERS_CIVIL_GENERAL_APPLICATIONS_INTERVAL: "6 years" + DELETE_OLD_LETTERS_CIVIL_SERVICE_INTERVAL: "6 years" + DELETE_OLD_LETTERS_CMC_CLAIM_STORE_INTERVAL: "2 years" + DELETE_OLD_LETTERS_DIVORCE_FRONTEND_INTERVAL: "3 months" + DELETE_OLD_LETTERS_FINREM_CASE_ORCHESTRATION_INTERVAL: "3 months" + DELETE_OLD_LETTERS_FINREM_DOCUMENT_GENERATOR_INTERVAL: "3 months" + DELETE_OLD_LETTERS_FPL_CASE_SERVICE_INTERVAL: "2 years" + DELETE_OLD_LETTERS_NFDIV_CASE_API_INTERVAL: "3 months" + DELETE_OLD_LETTERS_PRL_COS_API_INTERVAL: "18 years" + DELETE_OLD_LETTERS_PROBATE_BACKEND_INTERVAL: "1 year" + DELETE_OLD_LETTERS_SEND_LETTER_TESTS_INTERVAL: "2 years" + DELETE_OLD_LETTERS_SSCS_INTERVAL: "3 months" + # Don't modify below here image: 'hmctspublic.azurecr.io/rpe/send-letter-service:latest' diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java index c4646b1d6..1fafcbfcf 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -3,6 +3,7 @@ import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -28,10 +29,63 @@ public class DeleteOldLettersTask { private final LaunchDarklyClient launchDarklyClient; - private final Integer batchSize = 1000; + @Value("${delete-old-letters.batch-size:1000}") + private int batchSize; + + @Value("${delete-old-letters.civil-general-applications-interval:6 years}") + private String civilGeneralApplicationsInterval; + + @Value("${delete-old-letters.civil-service-interval:6 years}") + private String civilServiceInterval; + + @Value("${delete-old-letters.cmc-claim-store-interval:2 years}") + private String cmcClaimStoreInterval; + + @Value("${delete-old-letters.divorce-frontend-interval:3 months}") + private String divorceFrontendInterval; + + @Value("${delete-old-letters.finrem-case-orchestration-interval:3 months}") + private String finremCaseOrchestrationInterval; + + @Value("${delete-old-letters.finrem-document-generator-interval:3 months}") + private String finremDocumentGeneratorInterval; + + @Value("${delete-old-letters.fpl-case-service-interval:2 years}") + private String fplCaseServiceInterval; + + @Value("${delete-old-letters.nfdiv-case-api-interval:3 months}") + private String nfdivCaseApiInterval; + + @Value("${delete-old-letters.prl-cos-api-interval:18 years}") + private String prlCosApiInterval; + + @Value("${delete-old-letters.probate-backend-interval:1 year}") + private String probateBackendInterval; + + @Value("${delete-old-letters.send-letter-tests-interval:2 years}") + private String sendLetterTestsInterval; + + @Value("${delete-old-letters.sscs-interval:3 months}") + private String sscsInterval; // See V028__Add_batch_delete_letters_function.sql for sql function - private final String deleteQuery = "SELECT batch_delete_letters(?);"; + private final String deleteQuery = """ + SELECT batch_delete_letters( + ?, -- Batch size + CAST(? AS INTERVAL), -- civil_general_applications_interval + CAST(? AS INTERVAL), -- civil_service_interval + CAST(? AS INTERVAL), -- cmc_claim_store_interval + CAST(? AS INTERVAL), -- divorce_frontend_interval + CAST(? AS INTERVAL), -- finrem_case_orchestration_interval + CAST(? AS INTERVAL), -- finrem_document_generator_interval + CAST(? AS INTERVAL), -- fpl_case_service_interval + CAST(? AS INTERVAL), -- nfdiv_case_api_interval + CAST(? AS INTERVAL), -- prl_cos_api_interval + CAST(? AS INTERVAL), -- probate_backend_interval + CAST(? AS INTERVAL), -- send_letter_tests_interval + CAST(? AS INTERVAL) -- sscs_interval + ); + """; /** * Constructor for the DeleteOldLettersTask. @@ -46,8 +100,8 @@ public DeleteOldLettersTask(JdbcTemplate jdbcTemplate, LaunchDarklyClient launch * Deletes old letters from the database in batches based on the batch_delete_letters query. */ @SchedulerLock(name = TASK_NAME) - @Scheduled(cron = "0 0 17 ? * SAT", zone = EUROPE_LONDON) // Every Saturday at 5 PM - //@Scheduled(cron = "0 */2 * * * ?", zone = EUROPE_LONDON) // Every 2 minutes for testing + //@Scheduled(cron = "0 0 17 ? * SAT", zone = EUROPE_LONDON) // Every Saturday at 5 PM + @Scheduled(cron = "0 */2 * * * ?", zone = EUROPE_LONDON) // Every 2 minutes for testing public void run() { logger.info("Starting {} task", TASK_NAME); if (launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)) { @@ -59,7 +113,25 @@ public void run() { try { do { rowsDeleted = Optional.ofNullable( - jdbcTemplate.queryForObject(deleteQuery, new Object[]{batchSize}, Integer.class) + jdbcTemplate.queryForObject( + deleteQuery, + new Object[]{ + batchSize, + civilGeneralApplicationsInterval, // civil_general_applications_interval + civilServiceInterval, // civil_service_interval + cmcClaimStoreInterval, // cmc_claim_store_interval + divorceFrontendInterval, // divorce_frontend_interval + finremCaseOrchestrationInterval, // finrem_case_orchestration_interval + finremDocumentGeneratorInterval, // finrem_document_generator_interval + fplCaseServiceInterval, // fpl_case_service_interval + nfdivCaseApiInterval, // nfdiv_case_api_interval + prlCosApiInterval, // prl_cos_api_interval + probateBackendInterval, // probate_backend_interval + sendLetterTestsInterval, // send_letter_tests_interval + sscsInterval // sscs_interval + }, + Integer.class + ) ).orElse(0); totalRowsDeleted += rowsDeleted; logger.info("Batch deleted: {} rows, Total deleted: {} rows", rowsDeleted, totalRowsDeleted); diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 9866e3003..c87483567 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -205,3 +205,20 @@ documents: launchdarkly: sdk-key: ${LAUNCH_DARKLY_SDK_KEY:XXXXX} offline-mode: ${LAUNCH_DARKLY_OFFLINE_MODE:false} + +delete-old-letters: + batch-size: ${BATCH_SIZE:1000} + civil-general-applications-interval: ${CIVIL_GENERAL_APPLICATIONS_INTERVAL:6 years} + civil-service-interval: ${CIVIL_SERVICE_INTERVAL:6 years} + cmc-claim-store-interval: ${CMC_CLAIM_STORE_INTERVAL:2 years} + divorce-frontend-interval: ${DIVORCE_FRONTEND_INTERVAL:3 months} + finrem-case-orchestration-interval: ${FINREM_CASE_ORCHESTRATION_INTERVAL:3 months} + finrem-document-generator-interval: ${FINREM_DOCUMENT_GENERATOR_INTERVAL:3 months} + fpl-case-service-interval: ${FPL_CASE_SERVICE_INTERVAL:2 years} + nfdiv-case-api-interval: ${NFDIV_CASE_API_INTERVAL:3 months} + prl-cos-api-interval: ${PRL_COS_API_INTERVAL:18 years} + probate-backend-interval: ${PROBATE_BACKEND_INTERVAL:1 year} + send-letter-tests-interval: ${SEND_LETTER_TESTS_INTERVAL:2 years} + sscs-interval: ${SSCS_INTERVAL:3 months} + + diff --git a/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql b/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql index 1c60ccdcf..5a181041f 100644 --- a/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql +++ b/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql @@ -1,47 +1,62 @@ -CREATE OR REPLACE FUNCTION batch_delete_letters(batch_size INT) +CREATE OR REPLACE FUNCTION batch_delete_letters( + batch_size INT, + civil_general_applications_interval INTERVAL, + civil_service_interval INTERVAL, + cmc_claim_store_interval INTERVAL, + divorce_frontend_interval INTERVAL, + finrem_case_orchestration_interval INTERVAL, + finrem_document_generator_interval INTERVAL, + fpl_case_service_interval INTERVAL, + nfdiv_case_api_interval INTERVAL, + prl_cos_api_interval INTERVAL, + probate_backend_interval INTERVAL, + send_letter_tests_interval INTERVAL, + sscs_interval INTERVAL +) RETURNS INT AS $$ DECLARE rows_deleted INT; + BEGIN - -- Adding manual locks but have to lock each manually since we using delete cascade. - -- lock released automatically at end of transaction/commit. - LOCK TABLE letters, documents, letter_events IN ACCESS EXCLUSIVE MODE; + -- Adding manual locks but have to lock each manually since we are using DELETE CASCADE. + -- Lock is released automatically at the end of transaction/commit. + LOCK +TABLE letters, documents, letter_events IN ACCESS EXCLUSIVE MODE; - -- Perform deletion in a batch and get the number of rows deleted - WITH letters_to_delete AS ( - SELECT id -- Selecting indexed id column is all we need, for better performance - FROM letters - WHERE created_at < ( - CASE - WHEN service = 'civil_general_applications' THEN NOW() - INTERVAL '6 YEARS' - WHEN service = 'civil_service' THEN NOW() - INTERVAL '6 YEARS' - WHEN service = 'cmc_claim_store' THEN NOW() - INTERVAL '2 YEARS' - WHEN service = 'divorce_frontend' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'finrem_case_orchestration' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'finrem_document_generator' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'fpl_case_service' THEN NOW() - INTERVAL '2 YEARS' - WHEN service = 'nfdiv_case_api' THEN NOW() - INTERVAL '3 MONTHS' - WHEN service = 'prl_cos_api' THEN NOW() - INTERVAL '18 YEARS' - WHEN service = 'probate_backend' THEN NOW() - INTERVAL '1 YEAR' - WHEN service = 'send_letter_tests' THEN NOW() - INTERVAL '2 YEARS' - WHEN service = 'sscs' THEN NOW() - INTERVAL '3 MONTHS' - -- no default case - END - ) - -- For below don't delete old unprocessed / things that need investigating at some point - AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') - ORDER BY created_at ASC -- Prioritize oldest rows first - LIMIT batch_size -- Limit the number of rows selected in each batch - ) - DELETE FROM letters - USING letters_to_delete - WHERE letters.id = letters_to_delete.id; +-- Perform deletion in a batch and get the number of rows deleted +WITH letters_to_delete AS ( + SELECT id -- Selecting indexed id column is all we need, for better performance + FROM letters + WHERE created_at < ( + CASE + WHEN service = 'civil_general_applications' THEN NOW() - civil_general_applications_interval + WHEN service = 'civil_service' THEN NOW() - civil_service_interval + WHEN service = 'cmc_claim_store' THEN NOW() - cmc_claim_store_interval + WHEN service = 'divorce_frontend' THEN NOW() - divorce_frontend_interval + WHEN service = 'finrem_case_orchestration' THEN NOW() - finrem_case_orchestration_interval + WHEN service = 'finrem_document_generator' THEN NOW() - finrem_document_generator_interval + WHEN service = 'fpl_case_service' THEN NOW() - fpl_case_service_interval + WHEN service = 'nfdiv_case_api' THEN NOW() - nfdiv_case_api_interval + WHEN service = 'prl_cos_api' THEN NOW() - prl_cos_api_interval + WHEN service = 'probate_backend' THEN NOW() - probate_backend_interval + WHEN service = 'send_letter_tests' THEN NOW() - send_letter_tests_interval + WHEN service = 'sscs' THEN NOW() - sscs_interval + -- no default case + END + ) + -- For below don''t delete old unprocessed / things that need investigating at some point + AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') + ORDER BY created_at ASC -- Prioritize oldest rows first + LIMIT batch_size -- Limit the number of rows selected in each batch +) +DELETE +FROM letters USING letters_to_delete +WHERE letters.id = letters_to_delete.id; - -- Get the number of rows deleted by the DELETE statement - GET DIAGNOSTICS rows_deleted = ROW_COUNT; +-- Get the number of rows deleted by the DELETE statement +GET DIAGNOSTICS rows_deleted = ROW_COUNT; - -- Return the number of rows deleted - RETURN rows_deleted; +RETURN rows_deleted; END; $$ LANGUAGE plpgsql; From 6a0c4fdb00232f7c5d9ebaa8ee763f9010fff9ee Mon Sep 17 00:00:00 2001 From: hmcts-jenkins-j-to-z <61242337+hmcts-jenkins-j-to-z[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 16:23:39 +0000 Subject: [PATCH 07/10] Bumping chart version/ fixing aliases --- charts/rpe-send-letter-service/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/rpe-send-letter-service/Chart.yaml b/charts/rpe-send-letter-service/Chart.yaml index d5e549172..9dc10ed83 100644 --- a/charts/rpe-send-letter-service/Chart.yaml +++ b/charts/rpe-send-letter-service/Chart.yaml @@ -1,7 +1,7 @@ name: rpe-send-letter-service apiVersion: v2 home: https://github.com/hmcts/send-letter-service -version: 0.4.25 +version: 0.4.26 description: HMCTS Send letter service maintainers: - name: HMCTS BSP Team From 9cd6262e5da0827475c397a7d26c4091ef966304 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Tue, 17 Dec 2024 13:04:54 +0000 Subject: [PATCH 08/10] changed cron to take config --- charts/rpe-send-letter-service/values.yaml | 15 --------------- .../sendletter/tasks/DeleteOldLettersTask.java | 5 ++--- src/main/resources/application.yaml | 2 ++ 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/charts/rpe-send-letter-service/values.yaml b/charts/rpe-send-letter-service/values.yaml index 9a52d82b6..9c41f2ce2 100644 --- a/charts/rpe-send-letter-service/values.yaml +++ b/charts/rpe-send-letter-service/values.yaml @@ -80,20 +80,5 @@ java: FTP_UPLOADED_LETTERS_SUMMARY_REPORT_ENABLED: "false" DUPLICATES_CUT_OFF_TIME: 1 - # delete old letters interval times - DELETE_OLD_LETTERS_BATCH_SIZE: 1000 - DELETE_OLD_LETTERS_CIVIL_GENERAL_APPLICATIONS_INTERVAL: "6 years" - DELETE_OLD_LETTERS_CIVIL_SERVICE_INTERVAL: "6 years" - DELETE_OLD_LETTERS_CMC_CLAIM_STORE_INTERVAL: "2 years" - DELETE_OLD_LETTERS_DIVORCE_FRONTEND_INTERVAL: "3 months" - DELETE_OLD_LETTERS_FINREM_CASE_ORCHESTRATION_INTERVAL: "3 months" - DELETE_OLD_LETTERS_FINREM_DOCUMENT_GENERATOR_INTERVAL: "3 months" - DELETE_OLD_LETTERS_FPL_CASE_SERVICE_INTERVAL: "2 years" - DELETE_OLD_LETTERS_NFDIV_CASE_API_INTERVAL: "3 months" - DELETE_OLD_LETTERS_PRL_COS_API_INTERVAL: "18 years" - DELETE_OLD_LETTERS_PROBATE_BACKEND_INTERVAL: "1 year" - DELETE_OLD_LETTERS_SEND_LETTER_TESTS_INTERVAL: "2 years" - DELETE_OLD_LETTERS_SSCS_INTERVAL: "3 months" - # Don't modify below here image: 'hmctspublic.azurecr.io/rpe/send-letter-service:latest' diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java index 1fafcbfcf..36836c3c3 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -100,8 +100,7 @@ public DeleteOldLettersTask(JdbcTemplate jdbcTemplate, LaunchDarklyClient launch * Deletes old letters from the database in batches based on the batch_delete_letters query. */ @SchedulerLock(name = TASK_NAME) - //@Scheduled(cron = "0 0 17 ? * SAT", zone = EUROPE_LONDON) // Every Saturday at 5 PM - @Scheduled(cron = "0 */2 * * * ?", zone = EUROPE_LONDON) // Every 2 minutes for testing + @Scheduled(cron = "${delete-old-letters.cron:0 0 17 ? * SAT}", zone = EUROPE_LONDON) // default: Every Saturday at 5 PM public void run() { logger.info("Starting {} task", TASK_NAME); if (launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)) { @@ -117,7 +116,7 @@ public void run() { deleteQuery, new Object[]{ batchSize, - civilGeneralApplicationsInterval, // civil_general_applications_interval + civilGeneralApplicationsInterval, // civil_general_applications_interval civilServiceInterval, // civil_service_interval cmcClaimStoreInterval, // cmc_claim_store_interval divorceFrontendInterval, // divorce_frontend_interval diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index c87483567..b4c5e42c3 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -206,7 +206,9 @@ launchdarkly: sdk-key: ${LAUNCH_DARKLY_SDK_KEY:XXXXX} offline-mode: ${LAUNCH_DARKLY_OFFLINE_MODE:false} +#(0 */2 * * * ?) replace cron with this to test, will run every 2 minutes delete-old-letters: + cron: ${DELETE_OLD_LETTERS_CRON:0 0 17 ? * SAT} batch-size: ${BATCH_SIZE:1000} civil-general-applications-interval: ${CIVIL_GENERAL_APPLICATIONS_INTERVAL:6 years} civil-service-interval: ${CIVIL_SERVICE_INTERVAL:6 years} From 5a416ce0ae2c78d7138a0f0f6f424764036374e3 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Tue, 17 Dec 2024 14:13:03 +0000 Subject: [PATCH 09/10] checkstylemain --- .../hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java index 36836c3c3..c2fd644e7 100644 --- a/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java +++ b/src/main/java/uk/gov/hmcts/reform/sendletter/tasks/DeleteOldLettersTask.java @@ -98,9 +98,10 @@ public DeleteOldLettersTask(JdbcTemplate jdbcTemplate, LaunchDarklyClient launch /** * Deletes old letters from the database in batches based on the batch_delete_letters query. + * default cron: Every Saturday at 5 PM */ @SchedulerLock(name = TASK_NAME) - @Scheduled(cron = "${delete-old-letters.cron:0 0 17 ? * SAT}", zone = EUROPE_LONDON) // default: Every Saturday at 5 PM + @Scheduled(cron = "${delete-old-letters.cron:0 0 17 ? * SAT}", zone = EUROPE_LONDON) public void run() { logger.info("Starting {} task", TASK_NAME); if (launchDarklyClient.isFeatureEnabled(SEND_LETTER_SERVICE_DELETE_LETTERS_CRON)) { From fbbc23a7ae7a47914a2cc3b73840aade6703c076 Mon Sep 17 00:00:00 2001 From: Imran Ali Date: Fri, 20 Dec 2024 13:03:46 +0000 Subject: [PATCH 10/10] remove the migration --- ...028__Add_batch_delete_letters_function.sql | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql diff --git a/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql b/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql deleted file mode 100644 index 5a181041f..000000000 --- a/src/main/resources/db/migration/V028__Add_batch_delete_letters_function.sql +++ /dev/null @@ -1,62 +0,0 @@ -CREATE OR REPLACE FUNCTION batch_delete_letters( - batch_size INT, - civil_general_applications_interval INTERVAL, - civil_service_interval INTERVAL, - cmc_claim_store_interval INTERVAL, - divorce_frontend_interval INTERVAL, - finrem_case_orchestration_interval INTERVAL, - finrem_document_generator_interval INTERVAL, - fpl_case_service_interval INTERVAL, - nfdiv_case_api_interval INTERVAL, - prl_cos_api_interval INTERVAL, - probate_backend_interval INTERVAL, - send_letter_tests_interval INTERVAL, - sscs_interval INTERVAL -) -RETURNS INT AS -$$ -DECLARE -rows_deleted INT; - -BEGIN - -- Adding manual locks but have to lock each manually since we are using DELETE CASCADE. - -- Lock is released automatically at the end of transaction/commit. - LOCK -TABLE letters, documents, letter_events IN ACCESS EXCLUSIVE MODE; - --- Perform deletion in a batch and get the number of rows deleted -WITH letters_to_delete AS ( - SELECT id -- Selecting indexed id column is all we need, for better performance - FROM letters - WHERE created_at < ( - CASE - WHEN service = 'civil_general_applications' THEN NOW() - civil_general_applications_interval - WHEN service = 'civil_service' THEN NOW() - civil_service_interval - WHEN service = 'cmc_claim_store' THEN NOW() - cmc_claim_store_interval - WHEN service = 'divorce_frontend' THEN NOW() - divorce_frontend_interval - WHEN service = 'finrem_case_orchestration' THEN NOW() - finrem_case_orchestration_interval - WHEN service = 'finrem_document_generator' THEN NOW() - finrem_document_generator_interval - WHEN service = 'fpl_case_service' THEN NOW() - fpl_case_service_interval - WHEN service = 'nfdiv_case_api' THEN NOW() - nfdiv_case_api_interval - WHEN service = 'prl_cos_api' THEN NOW() - prl_cos_api_interval - WHEN service = 'probate_backend' THEN NOW() - probate_backend_interval - WHEN service = 'send_letter_tests' THEN NOW() - send_letter_tests_interval - WHEN service = 'sscs' THEN NOW() - sscs_interval - -- no default case - END - ) - -- For below don''t delete old unprocessed / things that need investigating at some point - AND status IN ('Posted', 'PostedLocally', 'Aborted', 'Skipped') - ORDER BY created_at ASC -- Prioritize oldest rows first - LIMIT batch_size -- Limit the number of rows selected in each batch -) -DELETE -FROM letters USING letters_to_delete -WHERE letters.id = letters_to_delete.id; - --- Get the number of rows deleted by the DELETE statement -GET DIAGNOSTICS rows_deleted = ROW_COUNT; - -RETURN rows_deleted; -END; -$$ LANGUAGE plpgsql;