feat: improves email generation performance and memory use

packages/server/src/db/index.js

@@ -1617,6 +1617,43 @@ async function getSavedSearches(userId, paginationParams) {
     return response;
 }
 
+async function usersBySavedSearchCriteria() {
+    const response = await knex('grants_saved_searches')
+        .select(knex.raw(`
+            criteria,
+            json_agg(
+                distinct jsonb_build_object(
+                    'id', ${TABLES.grants_saved_searches}.id,
+                    'created_by', ${TABLES.grants_saved_searches}.created_by,
+                    'name', ${TABLES.grants_saved_searches}.name,
+                    'status', ${TABLES.email_subscriptions}.status,
+                    'notification_type', ${TABLES.email_subscriptions}.notification_type,
+                    'tenant_id', ${TABLES.users}.tenant_id,
+                    'email', ${TABLES.users}.email
+                )
+            )`))
+        .join(TABLES.users, `${TABLES.grants_saved_searches}.created_by`, '=', `${TABLES.users}.id`)
+        .leftJoin(
+            TABLES.email_subscriptions, (builder) => {
+                builder
+                    .on(`${TABLES.grants_saved_searches}.created_by`, '=', `${TABLES.email_subscriptions}.user_id`)
+                    .andOn(`${TABLES.email_subscriptions}.notification_type`, '=', knex.raw('?', [emailConstants.notificationType.grantDigest]));
+            },
+        )
+        .where((q) => {
+            q
+                .where(`${TABLES.email_subscriptions}.status`, `${emailConstants.emailSubscriptionStatus.subscribed}`)
+                .orWhereNull(`${TABLES.email_subscriptions}.status`);
+        })
+        .groupBy('criteria');
+
+    return response.rows;
+}
+
+async function getUnsubscribedUsersWithSavedSearches() {
+    console.log('foo');
+}
+
 /**
  * Retrieves all saved searches joined with user and digest subscription information
  * @param int                   userId (optional)
@@ -1710,6 +1747,8 @@ module.exports = {
     deleteSavedSearch,
     updateSavedSearch,
     getAllUserSavedSearches,
+    usersBySavedSearchCriteria,
+    getUnsubscribedUsersWithSavedSearches,
     formatSearchCriteriaToQueryFilters,
     getUsers,
     createUser,

packages/server/src/lib/email.js

@@ -262,28 +262,74 @@ async function getAndSendGrantForSavedSearch({
 
     // NOTE: can't pass this as a separate parameter as it exceeds the complexity limit of 5
     criteriaObj.openDate = openDate;
+    let response;
 
     // Only 30 grants are shown on any given email and 31 will trigger a place to click to see more
-    const response = await db.getGrantsNew(
-        criteriaObj,
-        await db.buildPaginationParams({ currentPage: 1, perPage: 31 }),
-        {},
-        userSavedSearch.tenantId,
-    );
+    try {
+        response = await db.getGrantsNew(
+            criteriaObj,
+            await db.buildPaginationParams({ currentPage: 1, perPage: 31 }),
+            {},
+            userSavedSearch.tenantId,
+        );
+    } catch (err) {
+        console.error(`Error getting grants for ${userSavedSearch.name}: ${err}`);
+        return;
+    }
 
-    return sendGrantDigest({
-        name: userSavedSearch.name,
-        matchedGrants: response.data,
-        recipients: [userSavedSearch.email],
-        openDate,
-    });
+    try {
+        await sendGrantDigest({
+            name: userSavedSearch.name,
+            matchedGrants: response.data,
+            recipients: [userSavedSearch.email],
+            openDate,
+        });
+    } catch (err) {
+        console.error(`Error sending grant digest for ${userSavedSearch.name}, ${userSavedSearch.email}: ${err}`);
+    }
 }
 
 async function buildAndSendUserSavedSearchGrantDigest(userId, openDate) {
+    const start = Date.now();
+    let str = 'ts | rss | heTot | heUs | ext | arrBuf\n';
+    fileSystem.appendFile('foobar.txt', str, () => {});
+    setInterval(() => {
+        const mu = process.memoryUsage();
+        const elapsedTimeInSecs = (Date.now() - start) / 1000;
+        const timeRounded = Math.round(elapsedTimeInSecs * 100) / 100;
+        str = `${timeRounded}`;
+        /*
+            1.27 rss 0.12
+            1.27 heapTotal 0.08
+            1.27 heapUsed 0.05
+            1.27 external 0.01
+            1.27 arrayBuffers 0
+        */
+        for (const field of Object.keys(mu)) {
+            // # bytes / KB / MB / GB
+            const gbNow = mu[field] / 1024 / 1024 / 1024;
+            const gbRounded = Math.round(gbNow * 100) / 100;
+            str += ` | ${gbRounded}`;
+        }
+        str += '\n';
+        fileSystem.appendFile('foobar.txt', str, () => {}); // eslint-disable-line
+        str = '';
+    }, 500);
+
     if (!openDate) {
         openDate = moment().subtract(1, 'day').format('YYYY-MM-DD');
     }
     console.log(`Building and sending Grants Digest email for user: ${userId} on ${openDate}`);
+
+    const usersByCriteria = await db.usersBySavedSearchCriteria();
+    console.log(usersByCriteria);
+    // const unsubscribedUsersWithSavedSearches = new Set(await db.getUnsubscribedUsersWithSavedSearches());
+
+    // for (const [criteria, user_ids] of Object.entries(usersByCriteria)) {
+    //     user_ids.filter((user_id) => !unsubscribedUsersWithSavedSearches.has(user_id));
+    // }
+
+    // There's a limit to dd spans. Start a new-span per user or criteria.
     /*
     1. get all saved searches mapped to each user
     2. call getAndSendGrantForSavedSearch to find new grants and send the digest