From 0459b8ac939861877ab5fa092839a6d286b5e89f Mon Sep 17 00:00:00 2001 From: tzinckgraf Date: Tue, 21 Nov 2023 10:46:16 -0500 Subject: [PATCH 1/3] fix: #1676 - Cache prior reporting periods (#1857) * fix: #1676 - Cache prior reporting periods This is a first commit of a PR to cache prior reporting periods. This splits the report into two pieces such that the generate function should only need to call the most recent period. This should help with our reporting bottlenecks and with our performance issues. * fix: #1676 - Cache prior reporting periods Add caching for prior reporting periods. Added test cases as well. * fix: #1676 - Cache prior reporting periods Added linting changes * fix: #1676 - Cache prior reporting periods Incorporated the newest changes * fix: #1676 - Cache prior reporting periods Fixes for test case failures. Mostly error handling. * fix: #1676 - audit report caching * Fixed the cache to aggregate properly * fix: #1676 - audit report caching * functions now take prior data set and current data set * added more test cases * fix: #1676 - audit report caching * handled empty data better with nulls * reduced complexity and fixes based on code review * fix: #1676 - audit report caching * merged in production changes * fix: #1676 cache updates * fixes as per code review * ordering by date, not IDs * better default parameters * better fixture location * fix: #1676 - fixed undefined parameters * fix: #1676 - cache enhancements - added default parameter back in for tenant id --- .../arpa_reporter/server/fixtures/fixtures.js | 184 +++++++++++++++ .../server/lib/audit-report.spec.js | 40 ++++ .../services/generate-arpa-report.spec.js | 9 + .../src/arpa_reporter/lib/audit-report.js | 222 +++++++++++++++--- .../src/arpa_reporter/routes/audit-report.js | 19 ++ .../arpa_reporter/routes/reporting-periods.js | 8 + .../arpa_reporter/services/persist-upload.js | 13 + .../src/arpa_reporter/services/records.js | 14 +- 8 files changed, 467 insertions(+), 42 deletions(-) diff --git a/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js b/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js index d7a7b2c94..0d6739b80 100644 --- a/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js +++ b/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js @@ -103,12 +103,196 @@ const uploads = { }, }; +const audit_report_data = { + obligations: [{ + 'Reporting Period': 'Quarterly 1', + 'Period End Date': new Date(2021, 11, 31), + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')', + }, + 'Adopted Budget (EC tabs)': 300000, + 'Total Cumulative Obligations (EC tabs)': 300000, + 'Total Cumulative Expenditures (EC tabs)': 150000, + 'Current Period Obligations (EC tabs)': 150000, + 'Current Period Expenditures (EC tabs)': 150000, + 'Subaward Obligations (Subaward >50k)': 300000, + 'Total Expenditure Amount (Expenditures >50k)': 150000, + 'Current Period Obligations (Aggregate Awards <50k)': 0, + 'Current Period Expenditures (Aggregate Awards <50k)': 0, + }, { + 'Reporting Period': 'Quarterly 2', + 'Period End Date': new Date(2022, 2, 31), + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')', + }, + 'Adopted Budget (EC tabs)': 300000, + 'Total Cumulative Obligations (EC tabs)': 300000, + 'Total Cumulative Expenditures (EC tabs)': 150000, + 'Current Period Obligations (EC tabs)': 150000, + 'Current Period Expenditures (EC tabs)': 150000, + 'Subaward Obligations (Subaward >50k)': 300000, + 'Total Expenditure Amount (Expenditures >50k)': 150000, + 'Current Period Obligations (Aggregate Awards <50k)': 0, + 'Current Period Expenditures (Aggregate Awards <50k)': 0, + }, { + 'Reporting Period': 'Quarterly 3', + 'Period End Date': new Date(2022, 5, 30), + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')', + }, + 'Adopted Budget (EC tabs)': 300000, + 'Total Cumulative Obligations (EC tabs)': 300000, + 'Total Cumulative Expenditures (EC tabs)': 150000, + 'Current Period Obligations (EC tabs)': 150000, + 'Current Period Expenditures (EC tabs)': 150000, + 'Subaward Obligations (Subaward >50k)': 300000, + 'Total Expenditure Amount (Expenditures >50k)': 150000, + 'Current Period Obligations (Aggregate Awards <50k)': 0, + 'Current Period Expenditures (Aggregate Awards <50k)': 0, + }], + projectSummaries: [{ + 'Project ID': 6, + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')', + }, + 'Last Reported': 'Quarterly 1', + 'Adopted Budget': 300000, + 'Total Cumulative Obligations': 300000, + 'Total Cumulative Expenditures': 150000, + 'Current Period Obligations': 150000, + 'Current Period Expenditures': 150000, + 'Completion Status': 'Completed less than 50%', + }, { + 'Project ID': 7, + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')', + }, + 'Last Reported': 'Quarterly 2', + 'Adopted Budget': 300000, + 'Total Cumulative Obligations': 300000, + 'Total Cumulative Expenditures': 150000, + 'Current Period Obligations': 150000, + 'Current Period Expenditures': 150000, + 'Completion Status': 'Completed less than 50%', + }, + { + 'Project ID': 8, + Upload: { + f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')', + }, + 'Last Reported': 'Quarterly 3', + 'Adopted Budget': 300000, + 'Total Cumulative Obligations': 300000, + 'Total Cumulative Expenditures': 150000, + 'Current Period Obligations': 150000, + 'Current Period Expenditures': 150000, + 'Completion Status': 'Completed less than 50%', + }], + projectSummaryGroupedByProject: [{ + 'Project ID': '6', + 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', + 'Project Expenditure Category Group': '2-Negative Economic Impacts', + 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', + '2021-12-31 Total Aggregate Expenditures': 150000, + '2021-12-31 Total Expenditures for Awards Greater or Equal to $50k': 0, + '2021-12-31 Total Aggregate Obligations': 300000, + '2021-12-31 Total Obligations for Awards Greater or Equal to $50k': 0, + 'Capital Expenditure Amount': 0, + }, { + 'Project ID': '7', + 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', + 'Project Expenditure Category Group': '2-Negative Economic Impacts', + 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', + '2022-03-31 Total Aggregate Expenditures': 150000, + '2022-03-31 Total Expenditures for Awards Greater or Equal to $50k': 0, + '2022-03-31 Total Aggregate Obligations': 300000, + '2022-03-31 Total Obligations for Awards Greater or Equal to $50k': 0, + 'Capital Expenditure Amount': 0, + }, { + 'Project ID': '8', + 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', + 'Project Expenditure Category Group': '2-Negative Economic Impacts', + 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', + '2022-06-30 Total Aggregate Expenditures': 150000, + '2022-06-30 Total Expenditures for Awards Greater or Equal to $50k': 0, + '2022-06-30 Total Aggregate Obligations': 300000, + '2022-06-30 Total Obligations for Awards Greater or Equal to $50k': 0, + 'Capital Expenditure Amount': 0, + }], + KPIDataGroupedByProject: [{ + 'Project ID': '6', + 'Number of Subawards': 0, + 'Number of Expenditures': 1, + 'Evidence Based Total Spend': 0, + 'Evidence based total spend': null, + }, { + 'Project ID': '7', + 'Number of Subawards': 0, + 'Number of Expenditures': 1, + 'Evidence Based Total Spend': 0, + 'Evidence based total spend': null, + }, { + 'Project ID': '8', + 'Number of Subawards': 0, + 'Number of Expenditures': 1, + 'Evidence Based Total Spend': 0, + 'Evidence based total spend': null, + }], +}; + +const session = { + user: { + id: 1, + email: 'alex@usdigitalresponse.org', + name: 'Alex Allain', + role_id: 1, + role_name: 'admin', + role_rules: {}, + agency_id: 0, + agency_name: 'USDR', + agency_abbreviation: 'USDR', + agency_parent_id_id: null, + agency_warning_threshold: 30, + agency_danger_threshold: 15, + tenant_id: 1, + tenant_display_name: 'USDR Tenant', + tenant_main_agency_id: 400, + tenant_uses_spoc_process: false, + tags: null, + role: { id: 1, name: 'admin', rules: {} }, + agency: { + id: 0, + name: 'USDR', + abbreviation: 'USDR', + agency_parent_id: undefined, + warning_threshold: 30, + danger_threshold: 15, + main_agency_id: undefined, + subagencies: [], + }, + tenant: { + id: 1, + display_name: 'USDR Tenant', + main_agency_id: 400, + uses_spoc_process: false, + }, + emailPreferences: { + GRANT_ASSIGNMENT: 'SUBSCRIBED', + GRANT_INTEREST: 'SUBSCRIBED', + GRANT_DIGEST: 'SUBSCRIBED', + }, + }, + selectedAgency: 0, +}; + module.exports = { TABLES, reportingPeriods, uploads, TENANT_ID, users, + audit_report_data, + session, }; module.exports.clean = async (knex) => { diff --git a/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js b/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js index 6d22e1d7f..9b24697b6 100644 --- a/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js @@ -10,6 +10,7 @@ const email = require('../../../../src/lib/email'); const audit_report = require('../../../../src/arpa_reporter/lib/audit-report'); const aws = require('../../../../src/lib/gost-aws'); const { withTenantId } = require('../helpers/with-tenant-id'); +const { audit_report_data } = require('../fixtures/fixtures'); function handleUploadFake(type) { if (type === 'success') { @@ -29,6 +30,7 @@ describe('audit report generation', () => { afterEach(() => { sandbox.restore(); }); + it('sendEmailWithLink creates a presigned url and sends email to recipient', async () => { const sendFake = sandbox.fake.returns('foo'); sandbox.replace(email, 'sendAsyncReportEmail', sendFake); @@ -79,6 +81,7 @@ describe('audit report generation', () => { expect(sendEmailFake.firstCall.firstArg).to.equal('0/99/example.xlsx'); expect(sendEmailFake.firstCall.args[1]).to.equal('foo@example.com'); }); + it('generateAndSendEmail does not send an email if upload fails', async () => { const sendFake = sandbox.fake.returns('foo'); sandbox.replace(email, 'sendAsyncReportEmail', sendFake); @@ -119,6 +122,43 @@ describe('audit report generation', () => { expect(sendEmailFake.notCalled).to.equal(true); }); + it('generate audit report components', async () => { + const allData = audit_report_data; + const cachedData = Object.keys(audit_report_data).reduce((x, y) => { x[y] = audit_report_data[y].slice(0, -1); return x; }, {}); + const dataWithCache = Object.keys(audit_report_data).reduce((x, y) => { x[y] = [audit_report_data[y][audit_report_data[y].length - 1]]; return x; }, {}); + const periodId = 1; + const tenantId = 0; + const domain = 'test'; + + const obligationStub = sandbox.stub(audit_report, 'getObligationData'); + obligationStub.returns(allData.obligations); + const obligationsNoCache = await audit_report.createObligationSheet(periodId, domain, tenantId, null); + obligationStub.returns(dataWithCache.obligations); + const obligationsWithCache = await audit_report.createObligationSheet(periodId, domain, tenantId, cachedData.obligations); + expect(JSON.stringify(obligationsNoCache)).to.equal(JSON.stringify(obligationsWithCache)); + + const projectSummariesStub = sandbox.stub(audit_report, 'getProjectSummariesData'); + projectSummariesStub.returns(allData.projectSummaries); + const projectSummariesNoCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, null); + projectSummariesStub.returns(dataWithCache.projectSummaries); + const projectSummariesWithCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, cachedData.projectSummaries); + expect(JSON.stringify(projectSummariesNoCache)).to.equal(JSON.stringify(projectSummariesWithCache)); + + const projectSummaryGroupedByProjectStub = sandbox.stub(audit_report, 'getReportsGroupedByProjectData'); + projectSummaryGroupedByProjectStub.returns(allData.projectSummaryGroupedByProject); + const projectSummaryGroupedByProjectNoCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, null); + projectSummaryGroupedByProjectStub.returns(dataWithCache.projectSummaryGroupedByProject); + const projectSummaryGroupedByProjectWithCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, cachedData.projectSummaryGroupedByProject); + expect(JSON.stringify(projectSummaryGroupedByProjectNoCache)).to.equal(JSON.stringify(projectSummaryGroupedByProjectWithCache)); + + const kpiDataStub = sandbox.stub(audit_report, 'getKpiDataGroupedByProjectData'); + kpiDataStub.returns(allData.KPIDataGroupedByProject); + const kpiDataNoCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, null); + kpiDataStub.returns(dataWithCache.KPIDataGroupedByProject); + const kpiDataWithCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, cachedData.KPIDataGroupedByProject); + expect(JSON.stringify(kpiDataNoCache)).to.equal(JSON.stringify(kpiDataWithCache)); + }); + it('headers should be in the proper order', () => { const projects = [{ '09-30-2021 Total Aggregate Expenditures': 150000, diff --git a/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js b/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js index d9711046a..e0fa7bfe5 100644 --- a/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js @@ -1,6 +1,7 @@ const assert = require('assert'); const { generateReport } = require('../../../../src/arpa_reporter/services/generate-arpa-report'); +const { generate } = require('../../../../src/arpa_reporter/lib/audit-report'); const { withTenantId } = require('../helpers/with-tenant-id'); describe('arpa report generation', () => { @@ -11,4 +12,12 @@ describe('arpa report generation', () => { }); }); +describe('audit report generation', () => { + it('generates a report', async () => { + const tenantId = 0; + const report = await withTenantId(tenantId, () => generate('http://localhost')); + assert.ok(report); + }); +}); + // NOTE: This file was copied from tests/server/services/generate-arpa-report.spec.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/lib/audit-report.js b/packages/server/src/arpa_reporter/lib/audit-report.js index 53a82349d..c8c49683e 100644 --- a/packages/server/src/arpa_reporter/lib/audit-report.js +++ b/packages/server/src/arpa_reporter/lib/audit-report.js @@ -1,14 +1,16 @@ const tracer = require('dd-trace'); const ps = require('node:process'); +const path = require('path'); const moment = require('moment'); const { v4 } = require('uuid'); const XLSX = require('xlsx'); +const fs = require('fs/promises'); const { PutObjectCommand } = require('@aws-sdk/client-s3'); const { log } = require('../../lib/logging'); const aws = require('../../lib/gost-aws'); const { ec } = require('./format'); -const { getPreviousReportingPeriods, getAllReportingPeriods } = require('../db/reporting-periods'); +const { getPreviousReportingPeriods, getAllReportingPeriods, getReportingPeriod } = require('../db/reporting-periods'); const { getCurrentReportingPeriodID } = require('../db/settings'); const { recordsForProject, recordsForReportingPeriod, recordsForUpload, EC_SHEET_TYPES, @@ -17,6 +19,7 @@ const { usedForTreasuryExport } = require('../db/uploads'); const { ARPA_REPORTER_BASE_URL } = require('../environment'); const email = require('../../lib/email'); const { useTenantId } = require('../use-request'); +const { cacheFSName } = require('../services/persist-upload'); const { getUser, knex } = require('../../db'); const REPORTING_DATE_FORMAT = 'MM-DD-yyyy'; @@ -58,10 +61,18 @@ function getUploadLink(domain, id, filename) { return { f: `=HYPERLINK("${domain}/uploads/${id}","${filename}")` }; } -async function createObligationSheet(periodId, domain, tenantId, logger = log) { +async function createObligationSheet(periodId, domain, tenantId, dataBefore = null, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const data = await module.exports.getObligationData(periodId, domain, tenantId, calculatePriorPeriods, logger); + return [...data, ...(dataBefore ?? [])].sort((a, b) => (a['Period End Date'] - b['Period End Date'])); +} + +async function getObligationData(periodId, domain, tenantId, calculatePriorPeriods = true, logger = log) { logger.info('building rows for spreadsheet'); // select active reporting periods and sort by date - const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); + const reportingPeriods = calculatePriorPeriods + ? await getPreviousReportingPeriods(periodId, undefined, tenantId) + : [await getReportingPeriod(periodId, undefined, tenantId)]; logger.fields.sheet.totalReportingPeriods = reportingPeriods.length; logger.info('retrieved previous reporting periods'); @@ -139,7 +150,13 @@ async function createObligationSheet(periodId, domain, tenantId, logger = log) { return rows; } -async function createProjectSummaries(periodId, domain, tenantId, logger = log) { +async function createProjectSummariesSheet(periodId, domain, tenantId, dataBefore = null, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const data = await module.exports.getProjectSummariesData(periodId, domain, tenantId, calculatePriorPeriods, logger); + return [...data, ...(dataBefore ?? [])].sort((a, b) => (a['Project ID'] - b['Project ID'])); +} + +async function getProjectSummariesData(periodId, domain, tenantId, calculatePriorPeriods, logger = log) { logger.info('building rows for spreadsheet'); const uploads = await knex('uploads') .select({ @@ -242,9 +259,33 @@ function getRecordsByProject(records) { }, {}); } -async function createReportsGroupedByProject(periodId, tenantId, dateFormat = REPORTING_DATE_FORMAT, logger = log) { +async function createReportsGroupedByProjectSheet(periodId, tenantId, dataBefore = null, dateFormat = REPORTING_DATE_FORMAT, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const projects = await module.exports.getReportsGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, dateFormat, logger); + // go through each one and combine the columns + if (dataBefore !== null && dataBefore.length > 0) { + const dataBeforeRemaining = [...dataBefore]; + for (let i = 0; i < projects.length; i += 1) { + // check if we have this elsewhere + const project = projects[i]; + const index = dataBefore.findIndex((x) => x['Project ID'] === project['Project ID']); + for (let y = 0; (index !== -1) && (y < dataBeforeRemaining.length); y += 1) { + if (dataBeforeRemaining[y]['Project ID'] === project['Project ID']) { + project['Capital Expenditure Amount'] += dataBeforeRemaining[y]['Capital Expenditure Amount'] ?? 0; + projects[i] = { ...dataBeforeRemaining[y], ...project }; + delete dataBeforeRemaining[y]; + } + } + } + return [...projects, ...dataBeforeRemaining.filter((x) => x)].sort((a, b) => (a['Project ID'] - b['Project ID'])); + } + + return projects; +} + +async function getReportsGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, dateFormat = REPORTING_DATE_FORMAT, logger = log) { logger.info('building rows for spreadsheet'); - const records = await recordsForProject(periodId, tenantId); + const records = await recordsForProject(periodId, tenantId, calculatePriorPeriods); logger.fields.sheet.totalRecords = records.length; logger.info('retrieved records for projects'); const recordsByProject = getRecordsByProject(records); @@ -315,9 +356,34 @@ async function createReportsGroupedByProject(periodId, tenantId, dateFormat = RE return rows; } -async function createKpiDataGroupedByProject(periodId, tenantId, logger = log) { +async function createKpiDataGroupedByProjectSheet(periodId, tenantId, dataBefore = null, logger = log) { + const calculatePriorPeriods = dataBefore == null; + const rows = await module.exports.getKpiDataGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, logger); + // go through each one and combine the columns + if (dataBefore != null && dataBefore.length > 0) { + const dataBeforeRemaining = [...dataBefore]; + for (let i = 0; i < rows.length; i += 1) { + // check if we have this elsewhere + const row = rows[i]; + const index = dataBefore.findIndex((x) => x['Project ID'] === row['Project ID']); + for (let y = 0; (index !== -1) && (y < dataBeforeRemaining.length); y += 1) { + if (dataBeforeRemaining[y]['Project ID'] === row['Project ID']) { + const rowBefore = dataBeforeRemaining[y]; + row['Number of Subawards'] += rowBefore['Number of Subawards']; + row['Number of Expenditures'] += rowBefore['Number of Expenditures']; + row['Evidence Based Total Spend'] += rowBefore['Evidence Based Total Spend']; + delete dataBeforeRemaining[y]; + } + } + } + return [...rows, ...dataBeforeRemaining.filter((x) => x)].sort((a, b) => (a['Project ID'] - b['Project ID'])); + } + return rows; +} + +async function getKpiDataGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, logger = log) { logger.info('building rows for spreadsheet'); - const records = await recordsForProject(periodId, tenantId); + const records = await recordsForProject(periodId, tenantId, calculatePriorPeriods); logger.fields.sheet.totalRecords = records.length; logger.info('retrieved records for project'); const recordsByProject = getRecordsByProject(records); @@ -410,7 +476,97 @@ function createHeadersProjectSummariesV2(projectSummaryGroupedByProject) { return headers; } -async function generate(requestHost, tenantId) { +async function runCache(domain, tenantId, reportingPeriod, periodId = null) { + if (reportingPeriod == null) { + const reportingPeriods = await getPreviousReportingPeriods(periodId); + const previousReportingPeriods = reportingPeriods.filter((p) => p.id !== periodId); + reportingPeriod = previousReportingPeriods + .reduce((a, b) => (a.id > b.id ? a : b)); + } + const cacheFilename = cacheFSName(reportingPeriod, tenantId); + const data = await module.exports.generateSheets(reportingPeriod.id, domain, tenantId, null); + const jsonData = JSON.stringify(data); + await fs.mkdir(path.dirname(cacheFilename), { recursive: true }); + await fs.writeFile(cacheFilename, jsonData, { flag: 'wx' }); + return data; +} + +function reviveDate(key, value) { + // Matches strings like "2022-08-25T09:39:19.288Z" + const isoDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; + return typeof value === 'string' && isoDateRegex.test(value) + ? new Date(value) + : value; +} + +async function getCache(periodId, domain, tenantId, force = false, logger = log) { + // check if the cache file exists. if not, let's generate it + const reportingPeriods = await getPreviousReportingPeriods(periodId); + const previousReportingPeriods = reportingPeriods.filter((p) => p.id !== periodId); + if (previousReportingPeriods.length === 0) { + return { }; + } + const mostRecentPreviousReportingPeriod = previousReportingPeriods + .reduce((a, b) => (moment(a.start_date) > moment(b.start_date) ? a : b)); + const cacheFilename = cacheFSName(mostRecentPreviousReportingPeriod, tenantId); + let data = { }; + try { + if (force) { + throw new Error('forcing the cache'); + } + const cacheData = await fs.readFile(cacheFilename, { encoding: 'utf-8' }); + data = JSON.parse(cacheData, reviveDate); + logger.info(`Cache hit for ${tenantId}`); + } catch (err) { + logger.info(`Cache miss for ${tenantId}`); + data = await runCache(domain, tenantId, mostRecentPreviousReportingPeriod); + } + return data; +} + +async function generateSheets(periodId, domain, tenantId = useTenantId(), dataBefore = null, logger = log) { + // generate sheets data + const obligations = await tracer.trace('createObligationSheet', + async () => createObligationSheet( + periodId, + domain, + tenantId, + dataBefore?.obligations, + logger.child({ sheet: { name: 'Obligations & Expenditures' } }), + )); + const projectSummaries = await tracer.trace('createProjectSummariesSheet', + async () => createProjectSummariesSheet( + periodId, + domain, + tenantId, + dataBefore?.projectSummaries, + logger.child({ sheet: { name: 'Project Summaries' } }), + )); + const projectSummaryGroupedByProject = await tracer.trace('createReportsGroupedByProjectSheet', + async () => createReportsGroupedByProjectSheet( + periodId, + tenantId, + dataBefore?.projectSummaryGroupedByProject, + REPORTING_DATE_FORMAT, + logger.child({ sheet: { name: 'Project Summaries V2' } }), + )); + const KPIDataGroupedByProject = await tracer.trace('createKpiDataGroupedByProject', + async () => createKpiDataGroupedByProjectSheet( + periodId, + tenantId, + dataBefore?.KPIDataGroupedByProject, + logger.child({ sheet: { name: 'KPI' } }), + )); + + return { + obligations, + projectSummaries, + projectSummaryGroupedByProject, + KPIDataGroupedByProject, + }; +} + +async function generate(requestHost, tenantId, cache = true) { const domain = ARPA_REPORTER_BASE_URL ?? requestHost; return tracer.trace('generate()', async () => { const periodId = await getCurrentReportingPeriodID(undefined, tenantId); @@ -419,34 +575,15 @@ async function generate(requestHost, tenantId) { }); logger.info('determined current reporting period ID for workbook'); - // generate sheets data - const obligations = await tracer.trace('createObligationSheet', - async () => createObligationSheet( - periodId, - domain, - tenantId, - logger.child({ sheet: { name: 'Obligations & Expenditures' } }), - )); - const projectSummaries = await tracer.trace('createProjectSummaries', - async () => createProjectSummaries( - periodId, - domain, - tenantId, - logger.child({ sheet: { name: 'Project Summaries' } }), - )); - const projectSummaryGroupedByProject = await tracer.trace('createReportsGroupedByProject', - async () => createReportsGroupedByProject( - periodId, - tenantId, - REPORTING_DATE_FORMAT, - logger.child({ sheet: { name: 'Project Summaries V2' } }), - )); - const KPIDataGroupedByProject = await tracer.trace('createKpiDataGroupedByProject', - async () => createKpiDataGroupedByProject( - periodId, - tenantId, - logger.child({ sheet: { name: 'KPI' } }), - )); + const dataBefore = cache + ? await module.exports.getCache(periodId, domain, tenantId, false, logger) + : null; + const { + obligations, + projectSummaries, + projectSummaryGroupedByProject, + KPIDataGroupedByProject, + } = await module.exports.generateSheets(periodId, domain, tenantId, dataBefore); // compose workbook const workbook = tracer.trace('compose-workbook', () => { @@ -568,6 +705,17 @@ module.exports = { processSQSMessageRequest, sendEmailWithLink, createHeadersProjectSummariesV2, + generateSheets, + getCache, + + createObligationSheet, + getObligationData, + createProjectSummariesSheet, + getProjectSummariesData, + createReportsGroupedByProjectSheet, + getReportsGroupedByProjectData, + createKpiDataGroupedByProjectSheet, + getKpiDataGroupedByProjectData, }; // NOTE: This file was copied from src/server/lib/audit-report.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/routes/audit-report.js b/packages/server/src/arpa_reporter/routes/audit-report.js index 0fb0e964a..6f781868c 100644 --- a/packages/server/src/arpa_reporter/routes/audit-report.js +++ b/packages/server/src/arpa_reporter/routes/audit-report.js @@ -105,6 +105,25 @@ router.get('/', requireUser, async (req, res) => { res.send(Buffer.from(report.outputWorkBook, 'binary')); }); +router.post('/refresh-cache', async (req, res) => { + console.log('/api/audit-report/refresh-cache POST'); + try { + await audit_report.runCache( + req.headers.host ?? '', + req.body.tenantId, + ); + console.log('Successfully cached report'); + } catch (error) { + // In addition to sending the error message in the 500 response, log the full error stacktrace + console.log(`Could not cache report`, error); + res.status(500).send(error.message); + return; + } + res.json({ + status: 'OK', + }); +}); + module.exports = router; /* * * * */ diff --git a/packages/server/src/arpa_reporter/routes/reporting-periods.js b/packages/server/src/arpa_reporter/routes/reporting-periods.js index 518354e93..6014629b4 100644 --- a/packages/server/src/arpa_reporter/routes/reporting-periods.js +++ b/packages/server/src/arpa_reporter/routes/reporting-periods.js @@ -31,6 +31,7 @@ const { usedForTreasuryExport } = require('../db/uploads'); const { ensureAsyncContext } = require('../lib/ensure-async-context'); const { revalidateUploads } = require('../services/revalidate-uploads'); +const { runCache } = require('../lib/audit-report'); router.get('/', requireUser, async (req, res) => { const periods = await getAllReportingPeriods(); @@ -54,6 +55,13 @@ router.post('/close', requireAdminUser, async (req, res) => { return; } + try { + runCache(req.headers.host ?? '', period); + } catch (err) { + res.status(500).json({ error: err.message }); + return; + } + res.json({ status: 'OK', }); diff --git a/packages/server/src/arpa_reporter/services/persist-upload.js b/packages/server/src/arpa_reporter/services/persist-upload.js index 6fbb7afbf..ec5dd5eff 100644 --- a/packages/server/src/arpa_reporter/services/persist-upload.js +++ b/packages/server/src/arpa_reporter/services/persist-upload.js @@ -16,6 +16,7 @@ const { createUpload } = require('../db/uploads'); const { TEMP_DIR, UPLOAD_DIR } = require('../environment'); const { log } = require('../lib/log'); const ValidationError = require('../lib/validation-error'); +const { useTenantId } = require('../use-request'); /** * Get the path to the upload file for the given upload @@ -42,6 +43,17 @@ const jsonFSName = (upload) => { return path.join(TEMP_DIR, upload.id[0], filename); }; +/** + * Get the path to the JSON file for cached reporting periods + * @param {object} reportingPeriod + * @param {int} tenantId + * @returns {string} +*/ +const cacheFSName = (reportingPeriod, tenantId = useTenantId()) => { + const filename = `${tenantId}.${reportingPeriod.id}.json`; + return path.join(TEMP_DIR, filename); +}; + /** * Attempt to parse the buffer as an XLSX file * @param {Buffer} buffer @@ -302,6 +314,7 @@ module.exports = { bufferForUpload, workbookForUpload, uploadFSName, + cacheFSName, }; // NOTE: This file was copied from src/server/services/persist-upload.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/services/records.js b/packages/server/src/arpa_reporter/services/records.js index 5d45cff7f..42ca817e2 100644 --- a/packages/server/src/arpa_reporter/services/records.js +++ b/packages/server/src/arpa_reporter/services/records.js @@ -2,7 +2,7 @@ const XLSX = require('xlsx'); const { merge } = require('lodash'); const { workbookForUpload } = require('./persist-upload'); -const { getPreviousReportingPeriods } = require('../db/reporting-periods'); +const { getPreviousReportingPeriods, getReportingPeriod } = require('../db/reporting-periods'); const { usedForTreasuryExport } = require('../db/uploads'); const { log } = require('../lib/log'); const { requiredArgument } = require('../lib/preconditions'); @@ -210,11 +210,13 @@ async function recordsForReportingPeriod(periodId, tenantId) { * Get the most recent, validated record for each unique project, as of the * specified reporting period. */ -async function mostRecentProjectRecords(periodId, tenantId) { +async function mostRecentProjectRecords(periodId, tenantId, calculatePriorPeriods) { log(`mostRecentProjectRecords(${periodId})`); requiredArgument(periodId, 'must specify periodId in mostRecentProjectRecords'); - const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); + const reportingPeriods = calculatePriorPeriods + ? await getPreviousReportingPeriods(periodId, undefined, tenantId) + : [await getReportingPeriod(periodId, undefined, tenantId)]; const allRecords = await Promise.all( reportingPeriods.map(({ id }) => recordsForReportingPeriod(id, tenantId)), @@ -236,11 +238,13 @@ async function mostRecentProjectRecords(periodId, tenantId) { return Object.values(latestProjectRecords); } -async function recordsForProject(periodId, tenantId) { +async function recordsForProject(periodId, tenantId, calculatePriorPeriods) { log(`recordsForProject`); requiredArgument(periodId, 'must specify periodId in mostRecentProjectRecords'); - const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); + const reportingPeriods = calculatePriorPeriods + ? await getPreviousReportingPeriods(periodId, undefined, tenantId) + : [await getReportingPeriod(periodId, undefined, tenantId)]; const allRecords = await Promise.all( reportingPeriods.map(({ id }) => recordsForReportingPeriod(id, tenantId)), From 30826a44e3729e27625fc9f51c50a30a7718982e Mon Sep 17 00:00:00 2001 From: Dave M <64168322+replicantSocks@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:50:41 -0500 Subject: [PATCH 2/3] feat: add/fix fields in csv export (#2180) * feat: add/fix fields in csv export * fix: column header flipped in test --------- --- packages/server/__tests__/api/grants.test.js | 19 +++++++++++--- packages/server/src/db/index.js | 27 +++++++++++++++++--- packages/server/src/lib/email.js | 1 + packages/server/src/routes/grants.js | 8 +++++- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/packages/server/__tests__/api/grants.test.js b/packages/server/__tests__/api/grants.test.js index 1fc79c6a4..c930d1228 100644 --- a/packages/server/__tests__/api/grants.test.js +++ b/packages/server/__tests__/api/grants.test.js @@ -478,7 +478,7 @@ describe('`/api/grants` endpoint', () => { 'Title', 'Viewed By', 'Interested Teams', - 'Status', + 'Opportunity Status', 'Opportunity Category', 'Cost Sharing', 'Award Ceiling', @@ -487,11 +487,24 @@ describe('`/api/grants` endpoint', () => { 'Agency Code', 'Grant Id', 'URL', + 'Funding Type', + 'Appropriations Bill', + 'Agency Code', + 'Eligibility', ]; + const txt = await response.text(); + const rows = txt.split('\n'); + expect(rows[0]).to.equal(expectedCsvHeaders.join(',')); + + const cells = rows[1].split(','); + const valMap = new Map([...Array(cells.length).keys()].map((i) => [expectedCsvHeaders[i], cells[i]])); - expect(txt.split('\n')[0]).to.equal(expectedCsvHeaders.join(',')); - expect(txt.split('\n')[1]).to.contain('HHS-2021-IHS-TPI-0001,Community Health Aide Program: Tribal Planning &'); + expect(valMap.get('Opportunity Number')).to.equal('HHS-2021-IHS-TPI-0001'); + expect(valMap.get('Title')).to.equal('Community Health Aide Program: Tribal Planning & Implementation'); + expect(valMap.get('Funding Type')).to.equal('Other'); + expect(valMap.get('Agency Code')).to.equal('HHS-IHS'); + expect(valMap.get('Eligibility')).to.equal('"Native American tribal organizations (other than Federally recognized tribal governments)|Others(see text field entitled ""Additional Information on Eligibility"" for clarification)|Native American tribal governments(Federally recognized)"'); }); it('produces same number of rows as grid', async () => { diff --git a/packages/server/src/db/index.js b/packages/server/src/db/index.js index c5c13a8c6..4b677eaa1 100755 --- a/packages/server/src/db/index.js +++ b/packages/server/src/db/index.js @@ -668,6 +668,23 @@ function validateSearchFilters(filters) { return errors; } +function addCsvData(qb) { + qb + .select(knex.raw(` + CASE + WHEN grants.funding_instrument_codes = 'G' THEN 'Grant' + WHEN grants.funding_instrument_codes = 'CA' THEN 'Cooperative Agreement' + WHEN grants.funding_instrument_codes = 'PC' THEN 'Procurement Contract' + ELSE 'Other' + END as funding_type + `)) + .select(knex.raw(`array_to_string(array_agg(${TABLES.eligibility_codes}.label), '|') AS eligibility`)) + .leftJoin( + `${TABLES.eligibility_codes}`, + `${TABLES.eligibility_codes}.code`, '=', knex.raw(`ANY(string_to_array(${TABLES.grants}.eligibility_codes, ' '))`), + ); +} + /* filters: { reviewStatuses: List[Enum['Applied', 'Not Applying', 'Interested']], @@ -689,15 +706,15 @@ function validateSearchFilters(filters) { tenantId: number agencyId: number */ -async function getGrantsNew(filters, paginationParams, orderingParams, tenantId, agencyId) { - console.log(JSON.stringify([filters, paginationParams, orderingParams, tenantId, agencyId])); +async function getGrantsNew(filters, paginationParams, orderingParams, tenantId, agencyId, toCsv) { + console.log(JSON.stringify([filters, paginationParams, orderingParams, tenantId, agencyId, toCsv])); const errors = validateSearchFilters(filters); if (errors.length > 0) { throw new Error(`Invalid filters: ${errors.join(', ')}`); } - const data = await knex(TABLES.grants) + const query = knex(TABLES.grants) .select([ 'grants.grant_id', 'grants.grant_number', @@ -767,6 +784,10 @@ async function getGrantsNew(filters, paginationParams, orderingParams, tenantId, 'grants.funding_instrument_codes', 'grants.bill', ); + if (toCsv) { + query.modify(addCsvData); + } + const data = await query; const fullCount = data.length > 0 ? data[0].full_count : 0; diff --git a/packages/server/src/lib/email.js b/packages/server/src/lib/email.js index 69b315d4c..ec9a070ea 100644 --- a/packages/server/src/lib/email.js +++ b/packages/server/src/lib/email.js @@ -269,6 +269,7 @@ async function getAndSendGrantForSavedSearch({ await db.buildPaginationParams({ currentPage: 1, perPage: 31 }), {}, userSavedSearch.tenantId, + false, ); return sendGrantDigest({ diff --git a/packages/server/src/routes/grants.js b/packages/server/src/routes/grants.js index 675b05685..de9f64572 100755 --- a/packages/server/src/routes/grants.js +++ b/packages/server/src/routes/grants.js @@ -80,6 +80,7 @@ router.get('/next', requireUser, async (req, res) => { orderingParams, user.tenant_id, user.agency_id, + false, ); return res.json(grants); @@ -121,6 +122,7 @@ router.get('/exportCSVNew', requireUser, async (req, res) => { orderingParams, user.tenant_id, user.agency_id, + true, ); // Generate CSV @@ -155,7 +157,7 @@ router.get('/exportCSVNew', requireUser, async (req, res) => { { key: 'title', header: 'Title' }, { key: 'viewed_by', header: 'Viewed By' }, { key: 'interested_agencies', header: 'Interested Teams' }, - { key: 'opportunity_status', header: 'Status' }, + { key: 'opportunity_status', header: 'Opportunity Status' }, { key: 'opportunity_category', header: 'Opportunity Category' }, { key: 'cost_sharing', header: 'Cost Sharing' }, { key: 'award_ceiling', header: 'Award Ceiling' }, @@ -164,6 +166,10 @@ router.get('/exportCSVNew', requireUser, async (req, res) => { { key: 'agency_code', header: 'Agency Code' }, { key: 'grant_id', header: 'Grant Id' }, { key: 'url', header: 'URL' }, + { key: 'funding_type', header: 'Funding Type' }, + { key: 'bill', header: 'Appropriations Bill' }, + { key: 'agency_code', header: 'Agency Code' }, + { key: 'eligibility', header: 'Eligibility' }, ], }); From c98cf19906605f6e83a1e05b4596e1cf3d3a3807 Mon Sep 17 00:00:00 2001 From: aditya Date: Tue, 21 Nov 2023 11:30:03 -0500 Subject: [PATCH 3/3] Revert "fix: #1676 - Cache prior reporting periods (#1857)" (#2243) This reverts commit 0459b8ac939861877ab5fa092839a6d286b5e89f. --- .../arpa_reporter/server/fixtures/fixtures.js | 184 --------------- .../server/lib/audit-report.spec.js | 40 ---- .../services/generate-arpa-report.spec.js | 9 - .../src/arpa_reporter/lib/audit-report.js | 222 +++--------------- .../src/arpa_reporter/routes/audit-report.js | 19 -- .../arpa_reporter/routes/reporting-periods.js | 8 - .../arpa_reporter/services/persist-upload.js | 13 - .../src/arpa_reporter/services/records.js | 14 +- 8 files changed, 42 insertions(+), 467 deletions(-) diff --git a/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js b/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js index 0d6739b80..d7a7b2c94 100644 --- a/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js +++ b/packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js @@ -103,196 +103,12 @@ const uploads = { }, }; -const audit_report_data = { - obligations: [{ - 'Reporting Period': 'Quarterly 1', - 'Period End Date': new Date(2021, 11, 31), - Upload: { - f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')', - }, - 'Adopted Budget (EC tabs)': 300000, - 'Total Cumulative Obligations (EC tabs)': 300000, - 'Total Cumulative Expenditures (EC tabs)': 150000, - 'Current Period Obligations (EC tabs)': 150000, - 'Current Period Expenditures (EC tabs)': 150000, - 'Subaward Obligations (Subaward >50k)': 300000, - 'Total Expenditure Amount (Expenditures >50k)': 150000, - 'Current Period Obligations (Aggregate Awards <50k)': 0, - 'Current Period Expenditures (Aggregate Awards <50k)': 0, - }, { - 'Reporting Period': 'Quarterly 2', - 'Period End Date': new Date(2022, 2, 31), - Upload: { - f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')', - }, - 'Adopted Budget (EC tabs)': 300000, - 'Total Cumulative Obligations (EC tabs)': 300000, - 'Total Cumulative Expenditures (EC tabs)': 150000, - 'Current Period Obligations (EC tabs)': 150000, - 'Current Period Expenditures (EC tabs)': 150000, - 'Subaward Obligations (Subaward >50k)': 300000, - 'Total Expenditure Amount (Expenditures >50k)': 150000, - 'Current Period Obligations (Aggregate Awards <50k)': 0, - 'Current Period Expenditures (Aggregate Awards <50k)': 0, - }, { - 'Reporting Period': 'Quarterly 3', - 'Period End Date': new Date(2022, 5, 30), - Upload: { - f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')', - }, - 'Adopted Budget (EC tabs)': 300000, - 'Total Cumulative Obligations (EC tabs)': 300000, - 'Total Cumulative Expenditures (EC tabs)': 150000, - 'Current Period Obligations (EC tabs)': 150000, - 'Current Period Expenditures (EC tabs)': 150000, - 'Subaward Obligations (Subaward >50k)': 300000, - 'Total Expenditure Amount (Expenditures >50k)': 150000, - 'Current Period Obligations (Aggregate Awards <50k)': 0, - 'Current Period Expenditures (Aggregate Awards <50k)': 0, - }], - projectSummaries: [{ - 'Project ID': 6, - Upload: { - f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')', - }, - 'Last Reported': 'Quarterly 1', - 'Adopted Budget': 300000, - 'Total Cumulative Obligations': 300000, - 'Total Cumulative Expenditures': 150000, - 'Current Period Obligations': 150000, - 'Current Period Expenditures': 150000, - 'Completion Status': 'Completed less than 50%', - }, { - 'Project ID': 7, - Upload: { - f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')', - }, - 'Last Reported': 'Quarterly 2', - 'Adopted Budget': 300000, - 'Total Cumulative Obligations': 300000, - 'Total Cumulative Expenditures': 150000, - 'Current Period Obligations': 150000, - 'Current Period Expenditures': 150000, - 'Completion Status': 'Completed less than 50%', - }, - { - 'Project ID': 8, - Upload: { - f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')', - }, - 'Last Reported': 'Quarterly 3', - 'Adopted Budget': 300000, - 'Total Cumulative Obligations': 300000, - 'Total Cumulative Expenditures': 150000, - 'Current Period Obligations': 150000, - 'Current Period Expenditures': 150000, - 'Completion Status': 'Completed less than 50%', - }], - projectSummaryGroupedByProject: [{ - 'Project ID': '6', - 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', - 'Project Expenditure Category Group': '2-Negative Economic Impacts', - 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', - '2021-12-31 Total Aggregate Expenditures': 150000, - '2021-12-31 Total Expenditures for Awards Greater or Equal to $50k': 0, - '2021-12-31 Total Aggregate Obligations': 300000, - '2021-12-31 Total Obligations for Awards Greater or Equal to $50k': 0, - 'Capital Expenditure Amount': 0, - }, { - 'Project ID': '7', - 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', - 'Project Expenditure Category Group': '2-Negative Economic Impacts', - 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', - '2022-03-31 Total Aggregate Expenditures': 150000, - '2022-03-31 Total Expenditures for Awards Greater or Equal to $50k': 0, - '2022-03-31 Total Aggregate Obligations': 300000, - '2022-03-31 Total Obligations for Awards Greater or Equal to $50k': 0, - 'Capital Expenditure Amount': 0, - }, { - 'Project ID': '8', - 'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.', - 'Project Expenditure Category Group': '2-Negative Economic Impacts', - 'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance', - '2022-06-30 Total Aggregate Expenditures': 150000, - '2022-06-30 Total Expenditures for Awards Greater or Equal to $50k': 0, - '2022-06-30 Total Aggregate Obligations': 300000, - '2022-06-30 Total Obligations for Awards Greater or Equal to $50k': 0, - 'Capital Expenditure Amount': 0, - }], - KPIDataGroupedByProject: [{ - 'Project ID': '6', - 'Number of Subawards': 0, - 'Number of Expenditures': 1, - 'Evidence Based Total Spend': 0, - 'Evidence based total spend': null, - }, { - 'Project ID': '7', - 'Number of Subawards': 0, - 'Number of Expenditures': 1, - 'Evidence Based Total Spend': 0, - 'Evidence based total spend': null, - }, { - 'Project ID': '8', - 'Number of Subawards': 0, - 'Number of Expenditures': 1, - 'Evidence Based Total Spend': 0, - 'Evidence based total spend': null, - }], -}; - -const session = { - user: { - id: 1, - email: 'alex@usdigitalresponse.org', - name: 'Alex Allain', - role_id: 1, - role_name: 'admin', - role_rules: {}, - agency_id: 0, - agency_name: 'USDR', - agency_abbreviation: 'USDR', - agency_parent_id_id: null, - agency_warning_threshold: 30, - agency_danger_threshold: 15, - tenant_id: 1, - tenant_display_name: 'USDR Tenant', - tenant_main_agency_id: 400, - tenant_uses_spoc_process: false, - tags: null, - role: { id: 1, name: 'admin', rules: {} }, - agency: { - id: 0, - name: 'USDR', - abbreviation: 'USDR', - agency_parent_id: undefined, - warning_threshold: 30, - danger_threshold: 15, - main_agency_id: undefined, - subagencies: [], - }, - tenant: { - id: 1, - display_name: 'USDR Tenant', - main_agency_id: 400, - uses_spoc_process: false, - }, - emailPreferences: { - GRANT_ASSIGNMENT: 'SUBSCRIBED', - GRANT_INTEREST: 'SUBSCRIBED', - GRANT_DIGEST: 'SUBSCRIBED', - }, - }, - selectedAgency: 0, -}; - module.exports = { TABLES, reportingPeriods, uploads, TENANT_ID, users, - audit_report_data, - session, }; module.exports.clean = async (knex) => { diff --git a/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js b/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js index 9b24697b6..6d22e1d7f 100644 --- a/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js @@ -10,7 +10,6 @@ const email = require('../../../../src/lib/email'); const audit_report = require('../../../../src/arpa_reporter/lib/audit-report'); const aws = require('../../../../src/lib/gost-aws'); const { withTenantId } = require('../helpers/with-tenant-id'); -const { audit_report_data } = require('../fixtures/fixtures'); function handleUploadFake(type) { if (type === 'success') { @@ -30,7 +29,6 @@ describe('audit report generation', () => { afterEach(() => { sandbox.restore(); }); - it('sendEmailWithLink creates a presigned url and sends email to recipient', async () => { const sendFake = sandbox.fake.returns('foo'); sandbox.replace(email, 'sendAsyncReportEmail', sendFake); @@ -81,7 +79,6 @@ describe('audit report generation', () => { expect(sendEmailFake.firstCall.firstArg).to.equal('0/99/example.xlsx'); expect(sendEmailFake.firstCall.args[1]).to.equal('foo@example.com'); }); - it('generateAndSendEmail does not send an email if upload fails', async () => { const sendFake = sandbox.fake.returns('foo'); sandbox.replace(email, 'sendAsyncReportEmail', sendFake); @@ -122,43 +119,6 @@ describe('audit report generation', () => { expect(sendEmailFake.notCalled).to.equal(true); }); - it('generate audit report components', async () => { - const allData = audit_report_data; - const cachedData = Object.keys(audit_report_data).reduce((x, y) => { x[y] = audit_report_data[y].slice(0, -1); return x; }, {}); - const dataWithCache = Object.keys(audit_report_data).reduce((x, y) => { x[y] = [audit_report_data[y][audit_report_data[y].length - 1]]; return x; }, {}); - const periodId = 1; - const tenantId = 0; - const domain = 'test'; - - const obligationStub = sandbox.stub(audit_report, 'getObligationData'); - obligationStub.returns(allData.obligations); - const obligationsNoCache = await audit_report.createObligationSheet(periodId, domain, tenantId, null); - obligationStub.returns(dataWithCache.obligations); - const obligationsWithCache = await audit_report.createObligationSheet(periodId, domain, tenantId, cachedData.obligations); - expect(JSON.stringify(obligationsNoCache)).to.equal(JSON.stringify(obligationsWithCache)); - - const projectSummariesStub = sandbox.stub(audit_report, 'getProjectSummariesData'); - projectSummariesStub.returns(allData.projectSummaries); - const projectSummariesNoCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, null); - projectSummariesStub.returns(dataWithCache.projectSummaries); - const projectSummariesWithCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, cachedData.projectSummaries); - expect(JSON.stringify(projectSummariesNoCache)).to.equal(JSON.stringify(projectSummariesWithCache)); - - const projectSummaryGroupedByProjectStub = sandbox.stub(audit_report, 'getReportsGroupedByProjectData'); - projectSummaryGroupedByProjectStub.returns(allData.projectSummaryGroupedByProject); - const projectSummaryGroupedByProjectNoCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, null); - projectSummaryGroupedByProjectStub.returns(dataWithCache.projectSummaryGroupedByProject); - const projectSummaryGroupedByProjectWithCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, cachedData.projectSummaryGroupedByProject); - expect(JSON.stringify(projectSummaryGroupedByProjectNoCache)).to.equal(JSON.stringify(projectSummaryGroupedByProjectWithCache)); - - const kpiDataStub = sandbox.stub(audit_report, 'getKpiDataGroupedByProjectData'); - kpiDataStub.returns(allData.KPIDataGroupedByProject); - const kpiDataNoCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, null); - kpiDataStub.returns(dataWithCache.KPIDataGroupedByProject); - const kpiDataWithCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, cachedData.KPIDataGroupedByProject); - expect(JSON.stringify(kpiDataNoCache)).to.equal(JSON.stringify(kpiDataWithCache)); - }); - it('headers should be in the proper order', () => { const projects = [{ '09-30-2021 Total Aggregate Expenditures': 150000, diff --git a/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js b/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js index e0fa7bfe5..d9711046a 100644 --- a/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js +++ b/packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js @@ -1,7 +1,6 @@ const assert = require('assert'); const { generateReport } = require('../../../../src/arpa_reporter/services/generate-arpa-report'); -const { generate } = require('../../../../src/arpa_reporter/lib/audit-report'); const { withTenantId } = require('../helpers/with-tenant-id'); describe('arpa report generation', () => { @@ -12,12 +11,4 @@ describe('arpa report generation', () => { }); }); -describe('audit report generation', () => { - it('generates a report', async () => { - const tenantId = 0; - const report = await withTenantId(tenantId, () => generate('http://localhost')); - assert.ok(report); - }); -}); - // NOTE: This file was copied from tests/server/services/generate-arpa-report.spec.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/lib/audit-report.js b/packages/server/src/arpa_reporter/lib/audit-report.js index c8c49683e..53a82349d 100644 --- a/packages/server/src/arpa_reporter/lib/audit-report.js +++ b/packages/server/src/arpa_reporter/lib/audit-report.js @@ -1,16 +1,14 @@ const tracer = require('dd-trace'); const ps = require('node:process'); -const path = require('path'); const moment = require('moment'); const { v4 } = require('uuid'); const XLSX = require('xlsx'); -const fs = require('fs/promises'); const { PutObjectCommand } = require('@aws-sdk/client-s3'); const { log } = require('../../lib/logging'); const aws = require('../../lib/gost-aws'); const { ec } = require('./format'); -const { getPreviousReportingPeriods, getAllReportingPeriods, getReportingPeriod } = require('../db/reporting-periods'); +const { getPreviousReportingPeriods, getAllReportingPeriods } = require('../db/reporting-periods'); const { getCurrentReportingPeriodID } = require('../db/settings'); const { recordsForProject, recordsForReportingPeriod, recordsForUpload, EC_SHEET_TYPES, @@ -19,7 +17,6 @@ const { usedForTreasuryExport } = require('../db/uploads'); const { ARPA_REPORTER_BASE_URL } = require('../environment'); const email = require('../../lib/email'); const { useTenantId } = require('../use-request'); -const { cacheFSName } = require('../services/persist-upload'); const { getUser, knex } = require('../../db'); const REPORTING_DATE_FORMAT = 'MM-DD-yyyy'; @@ -61,18 +58,10 @@ function getUploadLink(domain, id, filename) { return { f: `=HYPERLINK("${domain}/uploads/${id}","${filename}")` }; } -async function createObligationSheet(periodId, domain, tenantId, dataBefore = null, logger = log) { - const calculatePriorPeriods = dataBefore == null; - const data = await module.exports.getObligationData(periodId, domain, tenantId, calculatePriorPeriods, logger); - return [...data, ...(dataBefore ?? [])].sort((a, b) => (a['Period End Date'] - b['Period End Date'])); -} - -async function getObligationData(periodId, domain, tenantId, calculatePriorPeriods = true, logger = log) { +async function createObligationSheet(periodId, domain, tenantId, logger = log) { logger.info('building rows for spreadsheet'); // select active reporting periods and sort by date - const reportingPeriods = calculatePriorPeriods - ? await getPreviousReportingPeriods(periodId, undefined, tenantId) - : [await getReportingPeriod(periodId, undefined, tenantId)]; + const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); logger.fields.sheet.totalReportingPeriods = reportingPeriods.length; logger.info('retrieved previous reporting periods'); @@ -150,13 +139,7 @@ async function getObligationData(periodId, domain, tenantId, calculatePriorPerio return rows; } -async function createProjectSummariesSheet(periodId, domain, tenantId, dataBefore = null, logger = log) { - const calculatePriorPeriods = dataBefore == null; - const data = await module.exports.getProjectSummariesData(periodId, domain, tenantId, calculatePriorPeriods, logger); - return [...data, ...(dataBefore ?? [])].sort((a, b) => (a['Project ID'] - b['Project ID'])); -} - -async function getProjectSummariesData(periodId, domain, tenantId, calculatePriorPeriods, logger = log) { +async function createProjectSummaries(periodId, domain, tenantId, logger = log) { logger.info('building rows for spreadsheet'); const uploads = await knex('uploads') .select({ @@ -259,33 +242,9 @@ function getRecordsByProject(records) { }, {}); } -async function createReportsGroupedByProjectSheet(periodId, tenantId, dataBefore = null, dateFormat = REPORTING_DATE_FORMAT, logger = log) { - const calculatePriorPeriods = dataBefore == null; - const projects = await module.exports.getReportsGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, dateFormat, logger); - // go through each one and combine the columns - if (dataBefore !== null && dataBefore.length > 0) { - const dataBeforeRemaining = [...dataBefore]; - for (let i = 0; i < projects.length; i += 1) { - // check if we have this elsewhere - const project = projects[i]; - const index = dataBefore.findIndex((x) => x['Project ID'] === project['Project ID']); - for (let y = 0; (index !== -1) && (y < dataBeforeRemaining.length); y += 1) { - if (dataBeforeRemaining[y]['Project ID'] === project['Project ID']) { - project['Capital Expenditure Amount'] += dataBeforeRemaining[y]['Capital Expenditure Amount'] ?? 0; - projects[i] = { ...dataBeforeRemaining[y], ...project }; - delete dataBeforeRemaining[y]; - } - } - } - return [...projects, ...dataBeforeRemaining.filter((x) => x)].sort((a, b) => (a['Project ID'] - b['Project ID'])); - } - - return projects; -} - -async function getReportsGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, dateFormat = REPORTING_DATE_FORMAT, logger = log) { +async function createReportsGroupedByProject(periodId, tenantId, dateFormat = REPORTING_DATE_FORMAT, logger = log) { logger.info('building rows for spreadsheet'); - const records = await recordsForProject(periodId, tenantId, calculatePriorPeriods); + const records = await recordsForProject(periodId, tenantId); logger.fields.sheet.totalRecords = records.length; logger.info('retrieved records for projects'); const recordsByProject = getRecordsByProject(records); @@ -356,34 +315,9 @@ async function getReportsGroupedByProjectData(periodId, tenantId, calculatePrior return rows; } -async function createKpiDataGroupedByProjectSheet(periodId, tenantId, dataBefore = null, logger = log) { - const calculatePriorPeriods = dataBefore == null; - const rows = await module.exports.getKpiDataGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, logger); - // go through each one and combine the columns - if (dataBefore != null && dataBefore.length > 0) { - const dataBeforeRemaining = [...dataBefore]; - for (let i = 0; i < rows.length; i += 1) { - // check if we have this elsewhere - const row = rows[i]; - const index = dataBefore.findIndex((x) => x['Project ID'] === row['Project ID']); - for (let y = 0; (index !== -1) && (y < dataBeforeRemaining.length); y += 1) { - if (dataBeforeRemaining[y]['Project ID'] === row['Project ID']) { - const rowBefore = dataBeforeRemaining[y]; - row['Number of Subawards'] += rowBefore['Number of Subawards']; - row['Number of Expenditures'] += rowBefore['Number of Expenditures']; - row['Evidence Based Total Spend'] += rowBefore['Evidence Based Total Spend']; - delete dataBeforeRemaining[y]; - } - } - } - return [...rows, ...dataBeforeRemaining.filter((x) => x)].sort((a, b) => (a['Project ID'] - b['Project ID'])); - } - return rows; -} - -async function getKpiDataGroupedByProjectData(periodId, tenantId, calculatePriorPeriods, logger = log) { +async function createKpiDataGroupedByProject(periodId, tenantId, logger = log) { logger.info('building rows for spreadsheet'); - const records = await recordsForProject(periodId, tenantId, calculatePriorPeriods); + const records = await recordsForProject(periodId, tenantId); logger.fields.sheet.totalRecords = records.length; logger.info('retrieved records for project'); const recordsByProject = getRecordsByProject(records); @@ -476,97 +410,7 @@ function createHeadersProjectSummariesV2(projectSummaryGroupedByProject) { return headers; } -async function runCache(domain, tenantId, reportingPeriod, periodId = null) { - if (reportingPeriod == null) { - const reportingPeriods = await getPreviousReportingPeriods(periodId); - const previousReportingPeriods = reportingPeriods.filter((p) => p.id !== periodId); - reportingPeriod = previousReportingPeriods - .reduce((a, b) => (a.id > b.id ? a : b)); - } - const cacheFilename = cacheFSName(reportingPeriod, tenantId); - const data = await module.exports.generateSheets(reportingPeriod.id, domain, tenantId, null); - const jsonData = JSON.stringify(data); - await fs.mkdir(path.dirname(cacheFilename), { recursive: true }); - await fs.writeFile(cacheFilename, jsonData, { flag: 'wx' }); - return data; -} - -function reviveDate(key, value) { - // Matches strings like "2022-08-25T09:39:19.288Z" - const isoDateRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; - return typeof value === 'string' && isoDateRegex.test(value) - ? new Date(value) - : value; -} - -async function getCache(periodId, domain, tenantId, force = false, logger = log) { - // check if the cache file exists. if not, let's generate it - const reportingPeriods = await getPreviousReportingPeriods(periodId); - const previousReportingPeriods = reportingPeriods.filter((p) => p.id !== periodId); - if (previousReportingPeriods.length === 0) { - return { }; - } - const mostRecentPreviousReportingPeriod = previousReportingPeriods - .reduce((a, b) => (moment(a.start_date) > moment(b.start_date) ? a : b)); - const cacheFilename = cacheFSName(mostRecentPreviousReportingPeriod, tenantId); - let data = { }; - try { - if (force) { - throw new Error('forcing the cache'); - } - const cacheData = await fs.readFile(cacheFilename, { encoding: 'utf-8' }); - data = JSON.parse(cacheData, reviveDate); - logger.info(`Cache hit for ${tenantId}`); - } catch (err) { - logger.info(`Cache miss for ${tenantId}`); - data = await runCache(domain, tenantId, mostRecentPreviousReportingPeriod); - } - return data; -} - -async function generateSheets(periodId, domain, tenantId = useTenantId(), dataBefore = null, logger = log) { - // generate sheets data - const obligations = await tracer.trace('createObligationSheet', - async () => createObligationSheet( - periodId, - domain, - tenantId, - dataBefore?.obligations, - logger.child({ sheet: { name: 'Obligations & Expenditures' } }), - )); - const projectSummaries = await tracer.trace('createProjectSummariesSheet', - async () => createProjectSummariesSheet( - periodId, - domain, - tenantId, - dataBefore?.projectSummaries, - logger.child({ sheet: { name: 'Project Summaries' } }), - )); - const projectSummaryGroupedByProject = await tracer.trace('createReportsGroupedByProjectSheet', - async () => createReportsGroupedByProjectSheet( - periodId, - tenantId, - dataBefore?.projectSummaryGroupedByProject, - REPORTING_DATE_FORMAT, - logger.child({ sheet: { name: 'Project Summaries V2' } }), - )); - const KPIDataGroupedByProject = await tracer.trace('createKpiDataGroupedByProject', - async () => createKpiDataGroupedByProjectSheet( - periodId, - tenantId, - dataBefore?.KPIDataGroupedByProject, - logger.child({ sheet: { name: 'KPI' } }), - )); - - return { - obligations, - projectSummaries, - projectSummaryGroupedByProject, - KPIDataGroupedByProject, - }; -} - -async function generate(requestHost, tenantId, cache = true) { +async function generate(requestHost, tenantId) { const domain = ARPA_REPORTER_BASE_URL ?? requestHost; return tracer.trace('generate()', async () => { const periodId = await getCurrentReportingPeriodID(undefined, tenantId); @@ -575,15 +419,34 @@ async function generate(requestHost, tenantId, cache = true) { }); logger.info('determined current reporting period ID for workbook'); - const dataBefore = cache - ? await module.exports.getCache(periodId, domain, tenantId, false, logger) - : null; - const { - obligations, - projectSummaries, - projectSummaryGroupedByProject, - KPIDataGroupedByProject, - } = await module.exports.generateSheets(periodId, domain, tenantId, dataBefore); + // generate sheets data + const obligations = await tracer.trace('createObligationSheet', + async () => createObligationSheet( + periodId, + domain, + tenantId, + logger.child({ sheet: { name: 'Obligations & Expenditures' } }), + )); + const projectSummaries = await tracer.trace('createProjectSummaries', + async () => createProjectSummaries( + periodId, + domain, + tenantId, + logger.child({ sheet: { name: 'Project Summaries' } }), + )); + const projectSummaryGroupedByProject = await tracer.trace('createReportsGroupedByProject', + async () => createReportsGroupedByProject( + periodId, + tenantId, + REPORTING_DATE_FORMAT, + logger.child({ sheet: { name: 'Project Summaries V2' } }), + )); + const KPIDataGroupedByProject = await tracer.trace('createKpiDataGroupedByProject', + async () => createKpiDataGroupedByProject( + periodId, + tenantId, + logger.child({ sheet: { name: 'KPI' } }), + )); // compose workbook const workbook = tracer.trace('compose-workbook', () => { @@ -705,17 +568,6 @@ module.exports = { processSQSMessageRequest, sendEmailWithLink, createHeadersProjectSummariesV2, - generateSheets, - getCache, - - createObligationSheet, - getObligationData, - createProjectSummariesSheet, - getProjectSummariesData, - createReportsGroupedByProjectSheet, - getReportsGroupedByProjectData, - createKpiDataGroupedByProjectSheet, - getKpiDataGroupedByProjectData, }; // NOTE: This file was copied from src/server/lib/audit-report.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/routes/audit-report.js b/packages/server/src/arpa_reporter/routes/audit-report.js index 6f781868c..0fb0e964a 100644 --- a/packages/server/src/arpa_reporter/routes/audit-report.js +++ b/packages/server/src/arpa_reporter/routes/audit-report.js @@ -105,25 +105,6 @@ router.get('/', requireUser, async (req, res) => { res.send(Buffer.from(report.outputWorkBook, 'binary')); }); -router.post('/refresh-cache', async (req, res) => { - console.log('/api/audit-report/refresh-cache POST'); - try { - await audit_report.runCache( - req.headers.host ?? '', - req.body.tenantId, - ); - console.log('Successfully cached report'); - } catch (error) { - // In addition to sending the error message in the 500 response, log the full error stacktrace - console.log(`Could not cache report`, error); - res.status(500).send(error.message); - return; - } - res.json({ - status: 'OK', - }); -}); - module.exports = router; /* * * * */ diff --git a/packages/server/src/arpa_reporter/routes/reporting-periods.js b/packages/server/src/arpa_reporter/routes/reporting-periods.js index 6014629b4..518354e93 100644 --- a/packages/server/src/arpa_reporter/routes/reporting-periods.js +++ b/packages/server/src/arpa_reporter/routes/reporting-periods.js @@ -31,7 +31,6 @@ const { usedForTreasuryExport } = require('../db/uploads'); const { ensureAsyncContext } = require('../lib/ensure-async-context'); const { revalidateUploads } = require('../services/revalidate-uploads'); -const { runCache } = require('../lib/audit-report'); router.get('/', requireUser, async (req, res) => { const periods = await getAllReportingPeriods(); @@ -55,13 +54,6 @@ router.post('/close', requireAdminUser, async (req, res) => { return; } - try { - runCache(req.headers.host ?? '', period); - } catch (err) { - res.status(500).json({ error: err.message }); - return; - } - res.json({ status: 'OK', }); diff --git a/packages/server/src/arpa_reporter/services/persist-upload.js b/packages/server/src/arpa_reporter/services/persist-upload.js index ec5dd5eff..6fbb7afbf 100644 --- a/packages/server/src/arpa_reporter/services/persist-upload.js +++ b/packages/server/src/arpa_reporter/services/persist-upload.js @@ -16,7 +16,6 @@ const { createUpload } = require('../db/uploads'); const { TEMP_DIR, UPLOAD_DIR } = require('../environment'); const { log } = require('../lib/log'); const ValidationError = require('../lib/validation-error'); -const { useTenantId } = require('../use-request'); /** * Get the path to the upload file for the given upload @@ -43,17 +42,6 @@ const jsonFSName = (upload) => { return path.join(TEMP_DIR, upload.id[0], filename); }; -/** - * Get the path to the JSON file for cached reporting periods - * @param {object} reportingPeriod - * @param {int} tenantId - * @returns {string} -*/ -const cacheFSName = (reportingPeriod, tenantId = useTenantId()) => { - const filename = `${tenantId}.${reportingPeriod.id}.json`; - return path.join(TEMP_DIR, filename); -}; - /** * Attempt to parse the buffer as an XLSX file * @param {Buffer} buffer @@ -314,7 +302,6 @@ module.exports = { bufferForUpload, workbookForUpload, uploadFSName, - cacheFSName, }; // NOTE: This file was copied from src/server/services/persist-upload.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z diff --git a/packages/server/src/arpa_reporter/services/records.js b/packages/server/src/arpa_reporter/services/records.js index 42ca817e2..5d45cff7f 100644 --- a/packages/server/src/arpa_reporter/services/records.js +++ b/packages/server/src/arpa_reporter/services/records.js @@ -2,7 +2,7 @@ const XLSX = require('xlsx'); const { merge } = require('lodash'); const { workbookForUpload } = require('./persist-upload'); -const { getPreviousReportingPeriods, getReportingPeriod } = require('../db/reporting-periods'); +const { getPreviousReportingPeriods } = require('../db/reporting-periods'); const { usedForTreasuryExport } = require('../db/uploads'); const { log } = require('../lib/log'); const { requiredArgument } = require('../lib/preconditions'); @@ -210,13 +210,11 @@ async function recordsForReportingPeriod(periodId, tenantId) { * Get the most recent, validated record for each unique project, as of the * specified reporting period. */ -async function mostRecentProjectRecords(periodId, tenantId, calculatePriorPeriods) { +async function mostRecentProjectRecords(periodId, tenantId) { log(`mostRecentProjectRecords(${periodId})`); requiredArgument(periodId, 'must specify periodId in mostRecentProjectRecords'); - const reportingPeriods = calculatePriorPeriods - ? await getPreviousReportingPeriods(periodId, undefined, tenantId) - : [await getReportingPeriod(periodId, undefined, tenantId)]; + const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); const allRecords = await Promise.all( reportingPeriods.map(({ id }) => recordsForReportingPeriod(id, tenantId)), @@ -238,13 +236,11 @@ async function mostRecentProjectRecords(periodId, tenantId, calculatePriorPeriod return Object.values(latestProjectRecords); } -async function recordsForProject(periodId, tenantId, calculatePriorPeriods) { +async function recordsForProject(periodId, tenantId) { log(`recordsForProject`); requiredArgument(periodId, 'must specify periodId in mostRecentProjectRecords'); - const reportingPeriods = calculatePriorPeriods - ? await getPreviousReportingPeriods(periodId, undefined, tenantId) - : [await getReportingPeriod(periodId, undefined, tenantId)]; + const reportingPeriods = await getPreviousReportingPeriods(periodId, undefined, tenantId); const allRecords = await Promise.all( reportingPeriods.map(({ id }) => recordsForReportingPeriod(id, tenantId)),