Prod Release 2023-10-03 - Take 1 (#2008)

packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js

@@ -1,3 +1,8 @@
+const chai = require('chai');
+const chaiAsPromised = require('chai-as-promised');
+
+chai.use(chaiAsPromised);
+
 const { expect } = require('chai');
 const sinon = require('sinon');
 const email = require('../../../../src/lib/email');
@@ -90,7 +95,10 @@ describe('audit report generation', () => {
         sandbox.replace(aws, 'getS3Client', s3Fake);
 
         const tenantId = 0;
-        await withTenantId(tenantId, () => audit_report.generateAndSendEmail('usdigitalresponse.org', '[email protected]'));
+        await expect(withTenantId(
+            tenantId,
+            () => audit_report.generateAndSendEmail('usdigitalresponse.org', '[email protected]'),
+        )).to.be.rejected;
 
         console.log('Asserting generate function');
         expect(generateFake.calledOnce).to.equal(true);
@@ -110,4 +118,59 @@ describe('audit report generation', () => {
         console.log('Asserting presigned and email function');
         expect(sendEmailFake.notCalled).to.equal(true);
     });
+
+    it('headers should be in the proper order', () => {
+        const projects = [{
+            '09-30-2021 Total Aggregate Expenditures': 150000,
+            '09-30-2022 Total Aggregate Obligations': 300000,
+            '09-30-2022 Total Expenditures for Awards Greater or Equal to $50k': 0,
+            '09-30-2022 Total Obligations for Awards Greater or Equal to $50k': 0,
+            'Capital Expenditure Amount': 0,
+            'Project Description': 'Sample description',
+            'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance',
+            'Project Expenditure Category Group': '2-Negative Economic Impacts',
+            'Project ID': '4',
+        }, {
+            '12-31-2022 Total Aggregate Expenditures': 150000,
+            '12-31-2022 Total Aggregate Obligations': 300000,
+            '12-31-2022 Total Expenditures for Awards Greater or Equal to $50k': 0,
+            '12-31-2022 Total Obligations for Awards Greater or Equal to $50k': 0,
+            'Capital Expenditure Amount': 0,
+            'Project Description': 'Sample description',
+            'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance',
+            'Project Expenditure Category Group': '2-Negative Economic Impacts',
+            'Project ID': '4',
+        }, {
+            '03-30-2023 Total Aggregate Expenditures': 150000,
+            '03-30-2023 Total Aggregate Obligations': 300000,
+            '03-30-2023 Total Expenditures for Awards Greater or Equal to $50k': 0,
+            '03-30-2023 Total Obligations for Awards Greater or Equal to $50k': 0,
+            'Capital Expenditure Amount': 0,
+            'Project Description': 'Sample description',
+            'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance',
+            'Project Expenditure Category Group': '2-Negative Economic Impacts',
+            'Project ID': '4',
+        }];
+        const headers = audit_report.createHeadersProjectSummariesV2(projects);
+        const headersExpected = [
+            'Project ID',
+            'Project Description',
+            'Project Expenditure Category Group',
+            'Project Expenditure Category',
+            'Capital Expenditure Amount',
+            '09-30-2021 Total Aggregate Obligations',
+            '12-31-2022 Total Aggregate Obligations',
+            '03-30-2023 Total Aggregate Obligations',
+            '09-30-2022 Total Aggregate Expenditures',
+            '12-31-2022 Total Aggregate Expenditures',
+            '03-30-2023 Total Aggregate Expenditures',
+            '09-30-2022 Total Obligations for Awards Greater or Equal to $50k',
+            '12-31-2022 Total Obligations for Awards Greater or Equal to $50k',
+            '03-30-2023 Total Obligations for Awards Greater or Equal to $50k',
+            '09-30-2022 Total Expenditures for Awards Greater or Equal to $50k',
+            '12-31-2022 Total Expenditures for Awards Greater or Equal to $50k',
+            '03-30-2023 Total Expenditures for Awards Greater or Equal to $50k',
+        ];
+        expect(headers[1]).to.equal(headersExpected[1]);
+    });
 });

packages/server/package.json

@@ -89,6 +89,7 @@
     "api": "^4.2.1",
     "aws-sdk-mock": "5.8.0",
     "chai": "^4.3.4",
+    "chai-as-promised": "^7.1.1",
     "chalk": "^5.0.1",
     "cookie-signature": "^1.2.0",
     "eslint": "^7.12.1",

packages/server/src/arpa_reporter/lib/audit-report.js

@@ -16,6 +16,9 @@ const email = require('../../lib/email');
 const { useTenantId } = require('../use-request');
 const { getUser } = require('../../db');
 
+const REPORTING_DATE_FORMAT = 'MM-DD-yyyy';
+const REPORTING_DATE_REGEX = /^(\d{2}-\d{2}-\d{4}) /;
+
 const log = (() => {
     const stickyData = {};
     return (msg, extra, sticky, clear) => {
@@ -236,7 +239,7 @@ function getRecordsByProject(records) {
     }, {});
 }
 
-async function createReportsGroupedByProject(periodId, tenantId) {
+async function createReportsGroupedByProject(periodId, tenantId, dateFormat = REPORTING_DATE_FORMAT) {
     log('called createReportsGroupedByProject()', { periodId, fn: 'createReportsGroupedByProject' });
     const records = await recordsForProject(periodId, tenantId);
     log('retrieved records for project', { fn: 'createReportsGroupedByProject', count: records.length });
@@ -274,7 +277,10 @@ async function createReportsGroupedByProject(periodId, tenantId) {
             fn: 'createReportsGroupedByProject', projectId,
         });
         allReportingPeriods.forEach((reportingPeriodId) => {
-            const reportingPeriodEndDate = reportingPeriods.filter((reportingPeriod) => reportingPeriod.id === reportingPeriodId)[0].end_date;
+            const reportingPeriodEndDate = moment(
+                reportingPeriods.filter((reportingPeriod) => reportingPeriod.id === reportingPeriodId)[0].end_date,
+                'yyyy-MM-DD',
+            ).format(dateFormat);
             [
                 `${reportingPeriodEndDate} Total Aggregate Expenditures`,
                 `${reportingPeriodEndDate} Total Expenditures for Awards Greater or Equal to $50k`,
@@ -289,9 +295,13 @@ async function createReportsGroupedByProject(periodId, tenantId) {
         log('setting values in each column', {
             fn: 'createReportsGroupedByProject', projectId,
         });
+
         projectRecords.forEach((r) => {
             // for project summaries v2 report
-            const reportingPeriodEndDate = reportingPeriods.filter((reportingPeriod) => r.upload.reporting_period_id === reportingPeriod.id)[0].end_date;
+            const reportingPeriodEndDate = moment(
+                reportingPeriods.filter((reportingPeriod) => r.upload.reporting_period_id === reportingPeriod.id)[0].end_date,
+                'yyyy-MM-DD',
+            ).format(dateFormat);
             row[`${reportingPeriodEndDate} Total Aggregate Expenditures`] += (r.content.Total_Expenditures__c || 0);
             row[`${reportingPeriodEndDate} Total Aggregate Obligations`] += (r.content.Total_Obligations__c || 0);
             row[`${reportingPeriodEndDate} Total Obligations for Awards Greater or Equal to $50k`] += (r.content.Award_Amount__c || 0);
@@ -338,6 +348,61 @@ async function createKpiDataGroupedByProject(periodId, tenantId) {
     });
 }
 
+/*
+ * Function to format the headers for the project summaries.
+ * The headers are split into the date and non-date headers.
+ * The non-date headers come first with an ordering, then the date headers.
+ */
+function createHeadersProjectSummariesV2(projectSummaryGroupedByProject) {
+    const keys = Array.from(new Set(projectSummaryGroupedByProject.map(Object.keys).flat()));
+    // split up by date and not date
+    const withDate = keys.filter((x) => REPORTING_DATE_REGEX.exec(x));
+    const withoutDate = keys.filter((x) => REPORTING_DATE_REGEX.exec(x) == null);
+
+    const dateOrder = withDate.reduce((x, y) => {
+        const key = y.replace(REPORTING_DATE_REGEX, '');
+        const value = y.match(REPORTING_DATE_REGEX)[1];
+        if (value == null) {
+            console.log('Could not find date in header');
+            return x;
+        }
+
+        if (!(key in x)) {
+            x[key] = [];
+        }
+        x[key].push(moment(value, REPORTING_DATE_FORMAT));
+        return x;
+    }, {});
+
+    const expectedOrderWithoutDate = [
+        'Project ID',
+        'Project Description',
+        'Project Expenditure Category Group',
+        'Project Expenditure Category',
+        'Capital Expenditure Amount',
+    ];
+
+    const expectedOrderWithDate = [
+        'Total Aggregate Obligations',
+        'Total Aggregate Expenditures',
+        'Total Obligations for Awards Greater or Equal to $50k',
+        'Total Expenditures for Awards Greater or Equal to $50k',
+    ];
+
+    // first add the properly ordered non-date headers,
+    // then add the headers sorted by the header group then the date
+    const headers = [
+        ...withoutDate.sort((p1, p2) => expectedOrderWithoutDate.indexOf(p1) - expectedOrderWithoutDate.indexOf(p2)),
+        ...Object.entries(dateOrder)
+            .sort((p1, p2) => expectedOrderWithDate.indexOf(p1[0]) - expectedOrderWithDate.indexOf(p2[0]))
+            .map((x) => x[1]
+                .sort((d1, d2) => d1.valueOf() - d2.valueOf())
+                .map((date) => `${date.format(REPORTING_DATE_FORMAT)} ${x[0]}`)).flat(),
+    ];
+
+    return headers;
+}
+
 async function generate(requestHost, tenantId) {
     log('called generate()');
     return tracer.trace('generate()', async () => {
@@ -365,22 +430,25 @@ async function generate(requestHost, tenantId) {
         log('composing workbook');
         const workbook = tracer.trace('compose-workbook', () => {
             // compose workbook
+
             const sheet1 = XLSX.utils.json_to_sheet(obligations, {
                 dateNF: 'MM/DD/YYYY',
             });
-            log('sheet 1 complete');
+            log('sheet 1 complete - Obligations & Expenditures');
             const sheet2 = XLSX.utils.json_to_sheet(projectSummaries, {
                 dateNF: 'MM/DD/YYYY',
             });
-            log('sheet 2 complete');
+            log('sheet 2 complete - Project Summaries');
+            const header = createHeadersProjectSummariesV2(projectSummaryGroupedByProject);
             const sheet3 = XLSX.utils.json_to_sheet(projectSummaryGroupedByProject, {
                 dateNF: 'MM/DD/YYYY',
+                header,
             });
-            log('sheet 3 complete');
+            log('sheet 3 complete - Project Summaries V2');
             const sheet4 = XLSX.utils.json_to_sheet(KPIDataGroupedByProject, {
                 dateNF: 'MM/DD/YYYY',
             });
-            log('sheet 4 complete');
+            log('sheet 4 complete - KPI');
             log('making new workbook');
             const newWorkbook = XLSX.utils.book_new();
             XLSX.utils.book_append_sheet(newWorkbook, sheet1, 'Obligations & Expenditures');
@@ -403,6 +471,7 @@ async function generate(requestHost, tenantId) {
 
         const filename = `audit-report-${moment().format('yy-MM-DD')}-${v4()}.xlsx`;
         log('generate() returning', {}, { generatedFilename: filename });
+
         return {
             periodId,
             filename,
@@ -434,14 +503,15 @@ async function generateAndSendEmail(requestHost, recipientEmail, tenantId = useT
         Body: report.outputWorkBook,
         ServerSideEncryption: 'AES256',
     };
+
     try {
-        console.log(uploadParams);
         console.log(uploadParams);
         log('uploading report', { bucket: uploadParams.Bucket, key: uploadParams.Key });
         await s3.send(new PutObjectCommand(uploadParams));
         await module.exports.sendEmailWithLink(reportKey, recipientEmail);
     } catch (err) {
         console.log(`Failed to upload/email audit report ${err}`);
+        throw err;
     }
     log('generateAndSendEmail() complete', null, null, true);
 }
@@ -460,7 +530,7 @@ async function processSQSMessageRequest(message) {
         if (!user) {
             throw new Error(`user not found: ${requestData.userId}`);
         }
-        generateAndSendEmail(ARPA_REPORTER_BASE_URL, user.email, user.tenant_id);
+        await generateAndSendEmail(ARPA_REPORTER_BASE_URL, user.email, user.tenant_id);
     } catch (e) {
         console.error('Failed to generate and send audit report', e);
         return false;
@@ -474,6 +544,7 @@ module.exports = {
     generateAndSendEmail,
     processSQSMessageRequest,
     sendEmailWithLink,
+    createHeadersProjectSummariesV2,
 };
 
 // NOTE: This file was copied from src/server/lib/audit-report.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z