diff --git a/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js b/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js index d7a7b2c947..0d6739b805 100644 --- a/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js +++ b/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js @@ -103,12 +103,196 @@ const uploads = { }, }; +const audit_report_data = { + obligations: [{ + 'Reporting Period': 'Quarterly 1', + 'Period End Date': new Date(2021, 11, 31), + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')', + }, + 'Adopted Budget (EC tabs)': 300000, + 'Total Cumulative Obligations (EC tabs)': 300000, + 'Total Cumulative Expenditures (EC tabs)': 150000, + 'Current Period Obligations (EC tabs)': 150000, + 'Current Period Expenditures (EC tabs)': 150000, + 'Subaward Obligations (Subaward >50k)': 300000, + 'Total Expenditure Amount (Expenditures >50k)': 150000, + 'Current Period Obligations (Aggregate Awards <50k)': 0, + 'Current Period Expenditures (Aggregate Awards <50k)': 0, + }, { + 'Reporting Period': 'Quarterly 2', + 'Period End Date': new Date(2022, 2, 31), + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')', + }, + 'Adopted Budget (EC tabs)': 300000, + 'Total Cumulative Obligations (EC tabs)': 300000, + 'Total Cumulative Expenditures (EC tabs)': 150000, + 'Current Period Obligations (EC tabs)': 150000, + 'Current Period Expenditures (EC tabs)': 150000, + 'Subaward Obligations (Subaward >50k)': 300000, + 'Total Expenditure Amount (Expenditures >50k)': 150000, + 'Current Period Obligations (Aggregate Awards <50k)': 0, + 'Current Period Expenditures (Aggregate Awards <50k)': 0, + }, { + 'Reporting Period': 'Quarterly 3', + 'Period End Date': new Date(2022, 5, 30), + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')', + }, + 'Adopted Budget (EC tabs)': 300000, + 'Total Cumulative Obligations (EC tabs)': 300000, + 'Total Cumulative Expenditures (EC tabs)': 150000, + 'Current Period Obligations (EC tabs)': 150000, + 'Current Period Expenditures (EC tabs)': 150000, + 'Subaward Obligations (Subaward >50k)': 300000, + 'Total Expenditure Amount (Expenditures >50k)': 150000, + 'Current Period Obligations (Aggregate Awards <50k)': 0, + 'Current Period Expenditures (Aggregate Awards <50k)': 0, + }], + projectSummaries: [{ + 'Project ID': 6, + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')', + }, + 'Last Reported': 'Quarterly 1', + 'Adopted Budget': 300000, + 'Total Cumulative Obligations': 300000, + 'Total Cumulative Expenditures': 150000, + 'Current Period Obligations': 150000, + 'Current Period Expenditures': 150000, + 'Completion Status': 'Completed less than 50%', + }, { + 'Project ID': 7, + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')', + }, + 'Last Reported': 'Quarterly 2', + 'Adopted Budget': 300000, + 'Total Cumulative Obligations': 300000, + 'Total Cumulative Expenditures': 150000, + 'Current Period Obligations': 150000, + 'Current Period Expenditures': 150000, + 'Completion Status': 'Completed less than 50%', + }, + { + 'Project ID': 8, + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')', + }, + 'Last Reported': 'Quarterly 3', + 'Adopted Budget': 300000, + 'Total Cumulative Obligations': 300000, + 'Total Cumulative Expenditures': 150000, + 'Current Period Obligations': 150000, + 'Current Period Expenditures': 150000, + 'Completion Status': 'Completed less than 50%', + }], + projectSummaryGroupedByProject: [{ + 'Project ID': '6', + 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', + 'Project Expenditure Category Group': '2-Negative Economic Impacts', + 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', + '2021-12-31 Total Aggregate Expenditures': 150000, + '2021-12-31 Total Expenditures for Awards Greater or Equal to $50k': 0, + '2021-12-31 Total Aggregate Obligations': 300000, + '2021-12-31 Total Obligations for Awards Greater or Equal to $50k': 0, + 'Capital Expenditure Amount': 0, + }, { + 'Project ID': '7', + 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', + 'Project Expenditure Category Group': '2-Negative Economic Impacts', + 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', + '2022-03-31 Total Aggregate Expenditures': 150000, + '2022-03-31 Total Expenditures for Awards Greater or Equal to $50k': 0, + '2022-03-31 Total Aggregate Obligations': 300000, + '2022-03-31 Total Obligations for Awards Greater or Equal to $50k': 0, + 'Capital Expenditure Amount': 0, + }, { + 'Project ID': '8', + 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', + 'Project Expenditure Category Group': '2-Negative Economic Impacts', + 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', + '2022-06-30 Total Aggregate Expenditures': 150000, + '2022-06-30 Total Expenditures for Awards Greater or Equal to $50k': 0, + '2022-06-30 Total Aggregate Obligations': 300000, + '2022-06-30 Total Obligations for Awards Greater or Equal to $50k': 0, + 'Capital Expenditure Amount': 0, + }], + KPIDataGroupedByProject: [{ + 'Project ID': '6', + 'Number of Subawards': 0, + 'Number of Expenditures': 1, + 'Evidence Based Total Spend': 0, + 'Evidence based total spend': null, + }, { + 'Project ID': '7', + 'Number of Subawards': 0, + 'Number of Expenditures': 1, + 'Evidence Based Total Spend': 0, + 'Evidence based total spend': null, + }, { + 'Project ID': '8', + 'Number of Subawards': 0, + 'Number of Expenditures': 1, + 'Evidence Based Total Spend': 0, + 'Evidence based total spend': null, + }], +}; + +const session = { + user: { + id: 1, + email: 'alex@usdigitalresponse.org', + name: 'Alex Allain', + role_id: 1, + role_name: 'admin', + role_rules: {}, + agency_id: 0, + agency_name: 'USDR', + agency_abbreviation: 'USDR', + agency_parent_id_id: null, + agency_warning_threshold: 30, + agency_danger_threshold: 15, + tenant_id: 1, + tenant_display_name: 'USDR Tenant', + tenant_main_agency_id: 400, + tenant_uses_spoc_process: false, + tags: null, + role: { id: 1, name: 'admin', rules: {} }, + agency: { + id: 0, + name: 'USDR', + abbreviation: 'USDR', + agency_parent_id: undefined, + warning_threshold: 30, + danger_threshold: 15, + main_agency_id: undefined, + subagencies: [], + }, + tenant: { + id: 1, + display_name: 'USDR Tenant', + main_agency_id: 400, + uses_spoc_process: false, + }, + emailPreferences: { + GRANT_ASSIGNMENT: 'SUBSCRIBED', + GRANT_INTEREST: 'SUBSCRIBED', + GRANT_DIGEST: 'SUBSCRIBED', + }, + }, + selectedAgency: 0, +}; + module.exports = { TABLES, reportingPeriods, uploads, TENANT_ID, users, + audit_report_data, + session, }; module.exports.clean = async (knex) => { diff --git a/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js b/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js index 6d22e1d7f4..9b24697b63 100644 --- a/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js @@ -10,6 +10,7 @@ const email = require('../../../../src/lib/email'); const audit_report = require('../../../../src/arpa_reporter/lib/audit-report'); const aws = require('../../../../src/lib/gost-aws'); const { withTenantId } = require('../helpers/with-tenant-id'); +const { audit_report_data } = require('../fixtures/fixtures'); function handleUploadFake(type) { if (type === 'success') { @@ -29,6 +30,7 @@ describe('audit report generation', () => { afterEach(() => { sandbox.restore(); }); + it('sendEmailWithLink creates a presigned url and sends email to recipient', async () => { const sendFake = sandbox.fake.returns('foo'); sandbox.replace(email, 'sendAsyncReportEmail', sendFake); @@ -79,6 +81,7 @@ describe('audit report generation', () => { expect(sendEmailFake.firstCall.firstArg).to.equal('0/99/example.xlsx'); expect(sendEmailFake.firstCall.args[1]).to.equal('foo@example.com'); }); + it('generateAndSendEmail does not send an email if upload fails', async () => { const sendFake = sandbox.fake.returns('foo'); sandbox.replace(email, 'sendAsyncReportEmail', sendFake); @@ -119,6 +122,43 @@ describe('audit report generation', () => { expect(sendEmailFake.notCalled).to.equal(true); }); + it('generate audit report components', async () => { + const allData = audit_report_data; + const cachedData = Object.keys(audit_report_data).reduce((x, y) => { x[y] = audit_report_data[y].slice(0, -1); return x; }, {}); + const dataWithCache = Object.keys(audit_report_data).reduce((x, y) => { x[y] = [audit_report_data[y][audit_report_data[y].length - 1]]; return x; }, {}); + const periodId = 1; + const tenantId = 0; + const domain = 'test'; + + const obligationStub = sandbox.stub(audit_report, 'getObligationData'); + obligationStub.returns(allData.obligations); + const obligationsNoCache = await audit_report.createObligationSheet(periodId, domain, tenantId, null); + obligationStub.returns(dataWithCache.obligations); + const obligationsWithCache = await audit_report.createObligationSheet(periodId, domain, tenantId, cachedData.obligations); + expect(JSON.stringify(obligationsNoCache)).to.equal(JSON.stringify(obligationsWithCache)); + + const projectSummariesStub = sandbox.stub(audit_report, 'getProjectSummariesData'); + projectSummariesStub.returns(allData.projectSummaries); + const projectSummariesNoCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, null); + projectSummariesStub.returns(dataWithCache.projectSummaries); + const projectSummariesWithCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, cachedData.projectSummaries); + expect(JSON.stringify(projectSummariesNoCache)).to.equal(JSON.stringify(projectSummariesWithCache)); + + const projectSummaryGroupedByProjectStub = sandbox.stub(audit_report, 'getReportsGroupedByProjectData'); + projectSummaryGroupedByProjectStub.returns(allData.projectSummaryGroupedByProject); + const projectSummaryGroupedByProjectNoCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, null); + projectSummaryGroupedByProjectStub.returns(dataWithCache.projectSummaryGroupedByProject); + const projectSummaryGroupedByProjectWithCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, cachedData.projectSummaryGroupedByProject); + expect(JSON.stringify(projectSummaryGroupedByProjectNoCache)).to.equal(JSON.stringify(projectSummaryGroupedByProjectWithCache)); + + const kpiDataStub = sandbox.stub(audit_report, 'getKpiDataGroupedByProjectData'); + kpiDataStub.returns(allData.KPIDataGroupedByProject); + const kpiDataNoCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, null); + kpiDataStub.returns(dataWithCache.KPIDataGroupedByProject); + const kpiDataWithCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, cachedData.KPIDataGroupedByProject); + expect(JSON.stringify(kpiDataNoCache)).to.equal(JSON.stringify(kpiDataWithCache)); + }); + it('headers should be in the proper order', () => { const projects = [{ '09-30-2021 Total Aggregate Expenditures': 150000, diff --git a/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js b/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js index d9711046ab..e0fa7bfe5d 100644 --- a/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js @@ -1,6 +1,7 @@ const assert = require('assert'); const { generateReport } = require('../../../../src/arpa_reporter/services/generate-arpa-report'); +const { generate } = require('../../../../src/arpa_reporter/lib/audit-report'); const { withTenantId } = require('../helpers/with-tenant-id'); describe('arpa report generation', () => { @@ -11,4 +12,12 @@ describe('arpa report generation', () => { }); }); +describe('audit report generation', () => { + it('generates a report', async () => { + const tenantId = 0; + const report = await withTenantId(tenantId, () => generate('http://localhost')); + assert.ok(report); + }); +}); + // NOTE: This file was copied from tests/server/services/generate-arpa-report.spec.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/lib/audit-report.js b/packages/server/src/arpa_reporter/lib/audit-report.js index 53a82349df..c8c49683e0 100644 --- a/packages/server/src/arpa_reporter/lib/audit-report.js +++ b/packages/server/src/arpa_reporter/lib/audit-report.js @@ -1,14 +1,16 @@ const tracer = require('dd-trace'); const ps = require('node:process'); +const path = require('path'); const moment = require('moment'); const { v4 } = require('uuid'); const XLSX = require('xlsx'); +const fs = require('fs/promises'); const { PutObjectCommand } = require('@aws-sdk/client-s3'); const { log } = require('../../lib/logging'); const aws = require('../../lib/gost-aws'); const { ec } = require('./format'); -const { getPreviousReportingPeriods, getAllReportingPeriods } = require('../db/reporting-periods'); +const { getPreviousReportingPeriods, getAllReportingPeriods, getReportingPeriod } = require('../db/reporting-periods'); const { getCurrentReportingPeriodID } = require('../db/settings'); const { recordsForProject, recordsForReportingPeriod, recordsForUpload, EC_SHEET_TYPES, @@ -17,6 +19,7 @@ const { usedForTreasuryExport } = require('../db/uploads'); const { ARPA_REPORTER_BASE_URL } = require('../environment'); const email = require('../../lib/email'); const { useTenantId } = require('../use-request'); +const { cacheFSName } = require('../services/persist-upload'); const { getUser, knex } = require('../../db'); const REPORTING_DATE_FORMAT = 'MM-DD-yyyy'; @@ -58,10 +61,18 @@ function getUploadLink(domain, id, filename) { return { f: `=HYPERLINK("${domain}/uploads/${id}","${filename}")` }; } -async function createObligationSheet(periodId, domain, tenantId, logger = log) { +async function createObligationSheet(periodId, domain, tenantId, dataBefore = null, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const data = await module.exports.getObligationData(periodId, domain, tenantId, calculatePriorPeriods, logger); + return [...data, ...(dataBefore ?? [])].sort((a, b) => (a['Period End Date'] - b['Period End Date'])); +} + +async function getObligationData(periodId, domain, tenantId, calculatePriorPeriods = true, logger = log) { logger.info('building rows for spreadsheet'); // select active reporting periods and sort by date - const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); + const reportingPeriods = calculatePriorPeriods + ? await getPreviousReportingPeriods(periodId, undefined, tenantId) + : [await getReportingPeriod(periodId, undefined, tenantId)]; logger.fields.sheet.totalReportingPeriods = reportingPeriods.length; logger.info('retrieved previous reporting periods'); @@ -139,7 +150,13 @@ async function createObligationSheet(periodId, domain, tenantId, logger = log) { return rows; } -async function createProjectSummaries(periodId, domain, tenantId, logger = log) { +async function createProjectSummariesSheet(periodId, domain, tenantId, dataBefore = null, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const data = await module.exports.getProjectSummariesData(periodId, domain, tenantId, calculatePriorPeriods, logger); + return [...data, ...(dataBefore ?? [])].sort((a, b) => (a['Project ID'] - b['Project ID'])); +} + +async function getProjectSummariesData(periodId, domain, tenantId, calculatePriorPeriods, logger = log) { logger.info('building rows for spreadsheet'); const uploads = await knex('uploads') .select({ @@ -242,9 +259,33 @@ function getRecordsByProject(records) { }, {}); } -async function createReportsGroupedByProject(periodId, tenantId, dateFormat = REPORTING_DATE_FORMAT, logger = log) { +async function createReportsGroupedByProjectSheet(periodId, tenantId, dataBefore = null, dateFormat = REPORTING_DATE_FORMAT, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const projects = await module.exports.getReportsGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, dateFormat, logger); + // go through each one and combine the columns + if (dataBefore !== null && dataBefore.length > 0) { + const dataBeforeRemaining = [...dataBefore]; + for (let i = 0; i < projects.length; i += 1) { + // check if we have this elsewhere + const project = projects[i]; + const index = dataBefore.findIndex((x) => x['Project ID'] === project['Project ID']); + for (let y = 0; (index !== -1) && (y < dataBeforeRemaining.length); y += 1) { + if (dataBeforeRemaining[y]['Project ID'] === project['Project ID']) { + project['Capital Expenditure Amount'] += dataBeforeRemaining[y]['Capital Expenditure Amount'] ?? 0; + projects[i] = { ...dataBeforeRemaining[y], ...project }; + delete dataBeforeRemaining[y]; + } + } + } + return [...projects, ...dataBeforeRemaining.filter((x) => x)].sort((a, b) => (a['Project ID'] - b['Project ID'])); + } + + return projects; +} + +async function getReportsGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, dateFormat = REPORTING_DATE_FORMAT, logger = log) { logger.info('building rows for spreadsheet'); - const records = await recordsForProject(periodId, tenantId); + const records = await recordsForProject(periodId, tenantId, calculatePriorPeriods); logger.fields.sheet.totalRecords = records.length; logger.info('retrieved records for projects'); const recordsByProject = getRecordsByProject(records); @@ -315,9 +356,34 @@ async function createReportsGroupedByProject(periodId, tenantId, dateFormat = RE return rows; } -async function createKpiDataGroupedByProject(periodId, tenantId, logger = log) { +async function createKpiDataGroupedByProjectSheet(periodId, tenantId, dataBefore = null, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const rows = await module.exports.getKpiDataGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, logger); + // go through each one and combine the columns + if (dataBefore != null && dataBefore.length > 0) { + const dataBeforeRemaining = [...dataBefore]; + for (let i = 0; i < rows.length; i += 1) { + // check if we have this elsewhere + const row = rows[i]; + const index = dataBefore.findIndex((x) => x['Project ID'] === row['Project ID']); + for (let y = 0; (index !== -1) && (y < dataBeforeRemaining.length); y += 1) { + if (dataBeforeRemaining[y]['Project ID'] === row['Project ID']) { + const rowBefore = dataBeforeRemaining[y]; + row['Number of Subawards'] += rowBefore['Number of Subawards']; + row['Number of Expenditures'] += rowBefore['Number of Expenditures']; + row['Evidence Based Total Spend'] += rowBefore['Evidence Based Total Spend']; + delete dataBeforeRemaining[y]; + } + } + } + return [...rows, ...dataBeforeRemaining.filter((x) => x)].sort((a, b) => (a['Project ID'] - b['Project ID'])); + } + return rows; +} + +async function getKpiDataGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, logger = log) { logger.info('building rows for spreadsheet'); - const records = await recordsForProject(periodId, tenantId); + const records = await recordsForProject(periodId, tenantId, calculatePriorPeriods); logger.fields.sheet.totalRecords = records.length; logger.info('retrieved records for project'); const recordsByProject = getRecordsByProject(records); @@ -410,7 +476,97 @@ function createHeadersProjectSummariesV2(projectSummaryGroupedByProject) { return headers; } -async function generate(requestHost, tenantId) { +async function runCache(domain, tenantId, reportingPeriod, periodId = null) { + if (reportingPeriod == null) { + const reportingPeriods = await getPreviousReportingPeriods(periodId); + const previousReportingPeriods = reportingPeriods.filter((p) => p.id !== periodId); + reportingPeriod = previousReportingPeriods + .reduce((a, b) => (a.id > b.id ? a : b)); + } + const cacheFilename = cacheFSName(reportingPeriod, tenantId); + const data = await module.exports.generateSheets(reportingPeriod.id, domain, tenantId, null); + const jsonData = JSON.stringify(data); + await fs.mkdir(path.dirname(cacheFilename), { recursive: true }); + await fs.writeFile(cacheFilename, jsonData, { flag: 'wx' }); + return data; +} + +function reviveDate(key, value) { + // Matches strings like "2022-08-25T09:39:19.288Z" + const isoDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; + return typeof value === 'string' && isoDateRegex.test(value) + ? new Date(value) + : value; +} + +async function getCache(periodId, domain, tenantId, force = false, logger = log) { + // check if the cache file exists. if not, let's generate it + const reportingPeriods = await getPreviousReportingPeriods(periodId); + const previousReportingPeriods = reportingPeriods.filter((p) => p.id !== periodId); + if (previousReportingPeriods.length === 0) { + return { }; + } + const mostRecentPreviousReportingPeriod = previousReportingPeriods + .reduce((a, b) => (moment(a.start_date) > moment(b.start_date) ? a : b)); + const cacheFilename = cacheFSName(mostRecentPreviousReportingPeriod, tenantId); + let data = { }; + try { + if (force) { + throw new Error('forcing the cache'); + } + const cacheData = await fs.readFile(cacheFilename, { encoding: 'utf-8' }); + data = JSON.parse(cacheData, reviveDate); + logger.info(`Cache hit for ${tenantId}`); + } catch (err) { + logger.info(`Cache miss for ${tenantId}`); + data = await runCache(domain, tenantId, mostRecentPreviousReportingPeriod); + } + return data; +} + +async function generateSheets(periodId, domain, tenantId = useTenantId(), dataBefore = null, logger = log) { + // generate sheets data + const obligations = await tracer.trace('createObligationSheet', + async () => createObligationSheet( + periodId, + domain, + tenantId, + dataBefore?.obligations, + logger.child({ sheet: { name: 'Obligations & Expenditures' } }), + )); + const projectSummaries = await tracer.trace('createProjectSummariesSheet', + async () => createProjectSummariesSheet( + periodId, + domain, + tenantId, + dataBefore?.projectSummaries, + logger.child({ sheet: { name: 'Project Summaries' } }), + )); + const projectSummaryGroupedByProject = await tracer.trace('createReportsGroupedByProjectSheet', + async () => createReportsGroupedByProjectSheet( + periodId, + tenantId, + dataBefore?.projectSummaryGroupedByProject, + REPORTING_DATE_FORMAT, + logger.child({ sheet: { name: 'Project Summaries V2' } }), + )); + const KPIDataGroupedByProject = await tracer.trace('createKpiDataGroupedByProject', + async () => createKpiDataGroupedByProjectSheet( + periodId, + tenantId, + dataBefore?.KPIDataGroupedByProject, + logger.child({ sheet: { name: 'KPI' } }), + )); + + return { + obligations, + projectSummaries, + projectSummaryGroupedByProject, + KPIDataGroupedByProject, + }; +} + +async function generate(requestHost, tenantId, cache = true) { const domain = ARPA_REPORTER_BASE_URL ?? requestHost; return tracer.trace('generate()', async () => { const periodId = await getCurrentReportingPeriodID(undefined, tenantId); @@ -419,34 +575,15 @@ async function generate(requestHost, tenantId) { }); logger.info('determined current reporting period ID for workbook'); - // generate sheets data - const obligations = await tracer.trace('createObligationSheet', - async () => createObligationSheet( - periodId, - domain, - tenantId, - logger.child({ sheet: { name: 'Obligations & Expenditures' } }), - )); - const projectSummaries = await tracer.trace('createProjectSummaries', - async () => createProjectSummaries( - periodId, - domain, - tenantId, - logger.child({ sheet: { name: 'Project Summaries' } }), - )); - const projectSummaryGroupedByProject = await tracer.trace('createReportsGroupedByProject', - async () => createReportsGroupedByProject( - periodId, - tenantId, - REPORTING_DATE_FORMAT, - logger.child({ sheet: { name: 'Project Summaries V2' } }), - )); - const KPIDataGroupedByProject = await tracer.trace('createKpiDataGroupedByProject', - async () => createKpiDataGroupedByProject( - periodId, - tenantId, - logger.child({ sheet: { name: 'KPI' } }), - )); + const dataBefore = cache + ? await module.exports.getCache(periodId, domain, tenantId, false, logger) + : null; + const { + obligations, + projectSummaries, + projectSummaryGroupedByProject, + KPIDataGroupedByProject, + } = await module.exports.generateSheets(periodId, domain, tenantId, dataBefore); // compose workbook const workbook = tracer.trace('compose-workbook', () => { @@ -568,6 +705,17 @@ module.exports = { processSQSMessageRequest, sendEmailWithLink, createHeadersProjectSummariesV2, + generateSheets, + getCache, + + createObligationSheet, + getObligationData, + createProjectSummariesSheet, + getProjectSummariesData, + createReportsGroupedByProjectSheet, + getReportsGroupedByProjectData, + createKpiDataGroupedByProjectSheet, + getKpiDataGroupedByProjectData, }; // NOTE: This file was copied from src/server/lib/audit-report.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/routes/audit-report.js b/packages/server/src/arpa_reporter/routes/audit-report.js index 0fb0e964ae..6f781868ca 100644 --- a/packages/server/src/arpa_reporter/routes/audit-report.js +++ b/packages/server/src/arpa_reporter/routes/audit-report.js @@ -105,6 +105,25 @@ router.get('/', requireUser, async (req, res) => { res.send(Buffer.from(report.outputWorkBook, 'binary')); }); +router.post('/refresh-cache', async (req, res) => { + console.log('/api/audit-report/refresh-cache POST'); + try { + await audit_report.runCache( + req.headers.host ?? '', + req.body.tenantId, + ); + console.log('Successfully cached report'); + } catch (error) { + // In addition to sending the error message in the 500 response, log the full error stacktrace + console.log(`Could not cache report`, error); + res.status(500).send(error.message); + return; + } + res.json({ + status: 'OK', + }); +}); + module.exports = router; /* * * * */ diff --git a/packages/server/src/arpa_reporter/routes/reporting-periods.js b/packages/server/src/arpa_reporter/routes/reporting-periods.js index 518354e93c..6014629b4c 100644 --- a/packages/server/src/arpa_reporter/routes/reporting-periods.js +++ b/packages/server/src/arpa_reporter/routes/reporting-periods.js @@ -31,6 +31,7 @@ const { usedForTreasuryExport } = require('../db/uploads'); const { ensureAsyncContext } = require('../lib/ensure-async-context'); const { revalidateUploads } = require('../services/revalidate-uploads'); +const { runCache } = require('../lib/audit-report'); router.get('/', requireUser, async (req, res) => { const periods = await getAllReportingPeriods(); @@ -54,6 +55,13 @@ router.post('/close', requireAdminUser, async (req, res) => { return; } + try { + runCache(req.headers.host ?? '', period); + } catch (err) { + res.status(500).json({ error: err.message }); + return; + } + res.json({ status: 'OK', }); diff --git a/packages/server/src/arpa_reporter/services/persist-upload.js b/packages/server/src/arpa_reporter/services/persist-upload.js index 6fbb7afbff..ec5dd5effe 100644 --- a/packages/server/src/arpa_reporter/services/persist-upload.js +++ b/packages/server/src/arpa_reporter/services/persist-upload.js @@ -16,6 +16,7 @@ const { createUpload } = require('../db/uploads'); const { TEMP_DIR, UPLOAD_DIR } = require('../environment'); const { log } = require('../lib/log'); const ValidationError = require('../lib/validation-error'); +const { useTenantId } = require('../use-request'); /** * Get the path to the upload file for the given upload @@ -42,6 +43,17 @@ const jsonFSName = (upload) => { return path.join(TEMP_DIR, upload.id[0], filename); }; +/** + * Get the path to the JSON file for cached reporting periods + * @param {object} reportingPeriod + * @param {int} tenantId + * @returns {string} +*/ +const cacheFSName = (reportingPeriod, tenantId = useTenantId()) => { + const filename = `${tenantId}.${reportingPeriod.id}.json`; + return path.join(TEMP_DIR, filename); +}; + /** * Attempt to parse the buffer as an XLSX file * @param {Buffer} buffer @@ -302,6 +314,7 @@ module.exports = { bufferForUpload, workbookForUpload, uploadFSName, + cacheFSName, }; // NOTE: This file was copied from src/server/services/persist-upload.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/services/records.js b/packages/server/src/arpa_reporter/services/records.js index 5d45cff7f8..42ca817e2e 100644 --- a/packages/server/src/arpa_reporter/services/records.js +++ b/packages/server/src/arpa_reporter/services/records.js @@ -2,7 +2,7 @@ const XLSX = require('xlsx'); const { merge } = require('lodash'); const { workbookForUpload } = require('./persist-upload'); -const { getPreviousReportingPeriods } = require('../db/reporting-periods'); +const { getPreviousReportingPeriods, getReportingPeriod } = require('../db/reporting-periods'); const { usedForTreasuryExport } = require('../db/uploads'); const { log } = require('../lib/log'); const { requiredArgument } = require('../lib/preconditions'); @@ -210,11 +210,13 @@ async function recordsForReportingPeriod(periodId, tenantId) { * Get the most recent, validated record for each unique project, as of the * specified reporting period. */ -async function mostRecentProjectRecords(periodId, tenantId) { +async function mostRecentProjectRecords(periodId, tenantId, calculatePriorPeriods) { log(`mostRecentProjectRecords(${periodId})`); requiredArgument(periodId, 'must specify periodId in mostRecentProjectRecords'); - const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); + const reportingPeriods = calculatePriorPeriods + ? await getPreviousReportingPeriods(periodId, undefined, tenantId) + : [await getReportingPeriod(periodId, undefined, tenantId)]; const allRecords = await Promise.all( reportingPeriods.map(({ id }) => recordsForReportingPeriod(id, tenantId)), @@ -236,11 +238,13 @@ async function mostRecentProjectRecords(periodId, tenantId) { return Object.values(latestProjectRecords); } -async function recordsForProject(periodId, tenantId) { +async function recordsForProject(periodId, tenantId, calculatePriorPeriods) { log(`recordsForProject`); requiredArgument(periodId, 'must specify periodId in mostRecentProjectRecords'); - const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); + const reportingPeriods = calculatePriorPeriods + ? await getPreviousReportingPeriods(periodId, undefined, tenantId) + : [await getReportingPeriod(periodId, undefined, tenantId)]; const allRecords = await Promise.all( reportingPeriods.map(({ id }) => recordsForReportingPeriod(id, tenantId)),