Skip to content

Commit

Permalink
fix: #1676 - Cache prior reporting periods (#1857)
Browse files Browse the repository at this point in the history 
* fix: #1676 - Cache prior reporting periods
This is a first commit of a PR to cache prior reporting periods.
This splits the report into two pieces such that the generate
function should only need to call the most recent period.
This should help with our reporting bottlenecks and with our performance issues.

* fix: #1676 - Cache prior reporting periods
Add caching for prior reporting periods.
Added test cases as well.

* fix: #1676 - Cache prior reporting periods
Added linting changes

* fix: #1676 - Cache prior reporting periods
Incorporated the newest changes

* fix: #1676 - Cache prior reporting periods
Fixes for test case failures. Mostly error handling.

* fix: #1676 - audit report caching

* Fixed the cache to aggregate properly

* fix: #1676 - audit report caching

* functions now take prior data set and current data set

* added more test cases

* fix: #1676 - audit report caching

* handled empty data better with nulls

* reduced complexity and fixes based on code review

* fix: #1676 - audit report caching

* merged in production changes

* fix: #1676 cache updates

* fixes as per code review

* ordering by date, not IDs

* better default parameters

* better fixture location

* fix: #1676

- fixed undefined parameters

* fix: #1676 - cache enhancements
- added default parameter back in for tenant id
  • Loading branch information
@tzinckgraf
tzinckgraf authored Nov 21, 2023
1 parent 09d9fb1 commit 0459b8a
Show file tree
Hide file tree
Showing 8 changed files with 467 additions and 42 deletions.
184 changes: 184 additions & 0 deletions packages/server/__tests__/arpa_reporter/server/fixtures/fixtures.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,196 @@ const uploads = {
},
};

const audit_report_data = {
obligations: [{
'Reporting Period': 'Quarterly 1',
'Period End Date': new Date(2021, 11, 31),
Upload: {
f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')',
},
'Adopted Budget (EC tabs)': 300000,
'Total Cumulative Obligations (EC tabs)': 300000,
'Total Cumulative Expenditures (EC tabs)': 150000,
'Current Period Obligations (EC tabs)': 150000,
'Current Period Expenditures (EC tabs)': 150000,
'Subaward Obligations (Subaward >50k)': 300000,
'Total Expenditure Amount (Expenditures >50k)': 150000,
'Current Period Obligations (Aggregate Awards <50k)': 0,
'Current Period Expenditures (Aggregate Awards <50k)': 0,
}, {
'Reporting Period': 'Quarterly 2',
'Period End Date': new Date(2022, 2, 31),
Upload: {
f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')',
},
'Adopted Budget (EC tabs)': 300000,
'Total Cumulative Obligations (EC tabs)': 300000,
'Total Cumulative Expenditures (EC tabs)': 150000,
'Current Period Obligations (EC tabs)': 150000,
'Current Period Expenditures (EC tabs)': 150000,
'Subaward Obligations (Subaward >50k)': 300000,
'Total Expenditure Amount (Expenditures >50k)': 150000,
'Current Period Obligations (Aggregate Awards <50k)': 0,
'Current Period Expenditures (Aggregate Awards <50k)': 0,
}, {
'Reporting Period': 'Quarterly 3',
'Period End Date': new Date(2022, 5, 30),
Upload: {
f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')',
},
'Adopted Budget (EC tabs)': 300000,
'Total Cumulative Obligations (EC tabs)': 300000,
'Total Cumulative Expenditures (EC tabs)': 150000,
'Current Period Obligations (EC tabs)': 150000,
'Current Period Expenditures (EC tabs)': 150000,
'Subaward Obligations (Subaward >50k)': 300000,
'Total Expenditure Amount (Expenditures >50k)': 150000,
'Current Period Obligations (Aggregate Awards <50k)': 0,
'Current Period Expenditures (Aggregate Awards <50k)': 0,
}],
projectSummaries: [{
'Project ID': 6,
Upload: {
f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/601a2011-91d5-4acb-b83e-f47ee8ae462f\',\'UPLOAD_CLEAN_ID6.xlsm\')',
},
'Last Reported': 'Quarterly 1',
'Adopted Budget': 300000,
'Total Cumulative Obligations': 300000,
'Total Cumulative Expenditures': 150000,
'Current Period Obligations': 150000,
'Current Period Expenditures': 150000,
'Completion Status': 'Completed less than 50%',
}, {
'Project ID': 7,
Upload: {
f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/3bb6dbb3-741c-43d1-957a-2d35dd1dd2a9\',\'UPLOAD_CLEAN_ID7.xlsm\')',
},
'Last Reported': 'Quarterly 2',
'Adopted Budget': 300000,
'Total Cumulative Obligations': 300000,
'Total Cumulative Expenditures': 150000,
'Current Period Obligations': 150000,
'Current Period Expenditures': 150000,
'Completion Status': 'Completed less than 50%',
},
{
'Project ID': 8,
Upload: {
f: '=HYPERLINK(\'http://localhost:8080/arpa_reporter/uploads/db5de7d3-4209-46d0-9479-8696491aadfc\',\'UPLOAD_CLEAN_ID8.xlsm\')',
},
'Last Reported': 'Quarterly 3',
'Adopted Budget': 300000,
'Total Cumulative Obligations': 300000,
'Total Cumulative Expenditures': 150000,
'Current Period Obligations': 150000,
'Current Period Expenditures': 150000,
'Completion Status': 'Completed less than 50%',
}],
projectSummaryGroupedByProject: [{
'Project ID': '6',
'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.',
'Project Expenditure Category Group': '2-Negative Economic Impacts',
'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance',
'2021-12-31 Total Aggregate Expenditures': 150000,
'2021-12-31 Total Expenditures for Awards Greater or Equal to $50k': 0,
'2021-12-31 Total Aggregate Obligations': 300000,
'2021-12-31 Total Obligations for Awards Greater or Equal to $50k': 0,
'Capital Expenditure Amount': 0,
}, {
'Project ID': '7',
'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.',
'Project Expenditure Category Group': '2-Negative Economic Impacts',
'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance',
'2022-03-31 Total Aggregate Expenditures': 150000,
'2022-03-31 Total Expenditures for Awards Greater or Equal to $50k': 0,
'2022-03-31 Total Aggregate Obligations': 300000,
'2022-03-31 Total Obligations for Awards Greater or Equal to $50k': 0,
'Capital Expenditure Amount': 0,
}, {
'Project ID': '8',
'Project Description': 'This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state. \'New Family Child Care Providers\' can be defined, for the purposes of this program, as providers who do not currently hold a license with the Department of Human Services, which may include providers who previously held licenses in good standing with the Department or providers new to the field entirely. This project will fund the start-up costs of new Family Child Care providers to open high-quality FCC options and increase the overall supply of child care in the state.',
'Project Expenditure Category Group': '2-Negative Economic Impacts',
'Project Expenditure Category': '2.32-Business Incubators and Start-Up or Expansion Assistance',
'2022-06-30 Total Aggregate Expenditures': 150000,
'2022-06-30 Total Expenditures for Awards Greater or Equal to $50k': 0,
'2022-06-30 Total Aggregate Obligations': 300000,
'2022-06-30 Total Obligations for Awards Greater or Equal to $50k': 0,
'Capital Expenditure Amount': 0,
}],
KPIDataGroupedByProject: [{
'Project ID': '6',
'Number of Subawards': 0,
'Number of Expenditures': 1,
'Evidence Based Total Spend': 0,
'Evidence based total spend': null,
}, {
'Project ID': '7',
'Number of Subawards': 0,
'Number of Expenditures': 1,
'Evidence Based Total Spend': 0,
'Evidence based total spend': null,
}, {
'Project ID': '8',
'Number of Subawards': 0,
'Number of Expenditures': 1,
'Evidence Based Total Spend': 0,
'Evidence based total spend': null,
}],
};

const session = {
user: {
id: 1,
email: '[email protected]',
name: 'Alex Allain',
role_id: 1,
role_name: 'admin',
role_rules: {},
agency_id: 0,
agency_name: 'USDR',
agency_abbreviation: 'USDR',
agency_parent_id_id: null,
agency_warning_threshold: 30,
agency_danger_threshold: 15,
tenant_id: 1,
tenant_display_name: 'USDR Tenant',
tenant_main_agency_id: 400,
tenant_uses_spoc_process: false,
tags: null,
role: { id: 1, name: 'admin', rules: {} },
agency: {
id: 0,
name: 'USDR',
abbreviation: 'USDR',
agency_parent_id: undefined,
warning_threshold: 30,
danger_threshold: 15,
main_agency_id: undefined,
subagencies: [],
},
tenant: {
id: 1,
display_name: 'USDR Tenant',
main_agency_id: 400,
uses_spoc_process: false,
},
emailPreferences: {
GRANT_ASSIGNMENT: 'SUBSCRIBED',
GRANT_INTEREST: 'SUBSCRIBED',
GRANT_DIGEST: 'SUBSCRIBED',
},
},
selectedAgency: 0,
};

module.exports = {
TABLES,
reportingPeriods,
uploads,
TENANT_ID,
users,
audit_report_data,
session,
};

module.exports.clean = async (knex) => {
Expand Down
40 changes: 40 additions & 0 deletions packages/server/__tests__/arpa_reporter/server/lib/audit-report.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const email = require('../../../../src/lib/email');
const audit_report = require('../../../../src/arpa_reporter/lib/audit-report');
const aws = require('../../../../src/lib/gost-aws');
const { withTenantId } = require('../helpers/with-tenant-id');
const { audit_report_data } = require('../fixtures/fixtures');

function handleUploadFake(type) {
if (type === 'success') {
Expand All @@ -29,6 +30,7 @@ describe('audit report generation', () => {
afterEach(() => {
sandbox.restore();
});

it('sendEmailWithLink creates a presigned url and sends email to recipient', async () => {
const sendFake = sandbox.fake.returns('foo');
sandbox.replace(email, 'sendAsyncReportEmail', sendFake);
Expand Down Expand Up @@ -79,6 +81,7 @@ describe('audit report generation', () => {
expect(sendEmailFake.firstCall.firstArg).to.equal('0/99/example.xlsx');
expect(sendEmailFake.firstCall.args[1]).to.equal('[email protected]');
});

it('generateAndSendEmail does not send an email if upload fails', async () => {
const sendFake = sandbox.fake.returns('foo');
sandbox.replace(email, 'sendAsyncReportEmail', sendFake);
Expand Down Expand Up @@ -119,6 +122,43 @@ describe('audit report generation', () => {
expect(sendEmailFake.notCalled).to.equal(true);
});

it('generate audit report components', async () => {
const allData = audit_report_data;
const cachedData = Object.keys(audit_report_data).reduce((x, y) => { x[y] = audit_report_data[y].slice(0, -1); return x; }, {});
const dataWithCache = Object.keys(audit_report_data).reduce((x, y) => { x[y] = [audit_report_data[y][audit_report_data[y].length - 1]]; return x; }, {});
const periodId = 1;
const tenantId = 0;
const domain = 'test';

const obligationStub = sandbox.stub(audit_report, 'getObligationData');
obligationStub.returns(allData.obligations);
const obligationsNoCache = await audit_report.createObligationSheet(periodId, domain, tenantId, null);
obligationStub.returns(dataWithCache.obligations);
const obligationsWithCache = await audit_report.createObligationSheet(periodId, domain, tenantId, cachedData.obligations);
expect(JSON.stringify(obligationsNoCache)).to.equal(JSON.stringify(obligationsWithCache));

const projectSummariesStub = sandbox.stub(audit_report, 'getProjectSummariesData');
projectSummariesStub.returns(allData.projectSummaries);
const projectSummariesNoCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, null);
projectSummariesStub.returns(dataWithCache.projectSummaries);
const projectSummariesWithCache = await audit_report.createProjectSummariesSheet(periodId, domain, tenantId, cachedData.projectSummaries);
expect(JSON.stringify(projectSummariesNoCache)).to.equal(JSON.stringify(projectSummariesWithCache));

const projectSummaryGroupedByProjectStub = sandbox.stub(audit_report, 'getReportsGroupedByProjectData');
projectSummaryGroupedByProjectStub.returns(allData.projectSummaryGroupedByProject);
const projectSummaryGroupedByProjectNoCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, null);
projectSummaryGroupedByProjectStub.returns(dataWithCache.projectSummaryGroupedByProject);
const projectSummaryGroupedByProjectWithCache = await audit_report.createReportsGroupedByProjectSheet(periodId, tenantId, cachedData.projectSummaryGroupedByProject);
expect(JSON.stringify(projectSummaryGroupedByProjectNoCache)).to.equal(JSON.stringify(projectSummaryGroupedByProjectWithCache));

const kpiDataStub = sandbox.stub(audit_report, 'getKpiDataGroupedByProjectData');
kpiDataStub.returns(allData.KPIDataGroupedByProject);
const kpiDataNoCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, null);
kpiDataStub.returns(dataWithCache.KPIDataGroupedByProject);
const kpiDataWithCache = await audit_report.createKpiDataGroupedByProjectSheet(periodId, tenantId, cachedData.KPIDataGroupedByProject);
expect(JSON.stringify(kpiDataNoCache)).to.equal(JSON.stringify(kpiDataWithCache));
});

it('headers should be in the proper order', () => {
const projects = [{
'09-30-2021 Total Aggregate Expenditures': 150000,
Expand Down
9 changes: 9 additions & 0 deletions packages/server/__tests__/arpa_reporter/server/services/generate-arpa-report.spec.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const assert = require('assert');

const { generateReport } = require('../../../../src/arpa_reporter/services/generate-arpa-report');
const { generate } = require('../../../../src/arpa_reporter/lib/audit-report');
const { withTenantId } = require('../helpers/with-tenant-id');

describe('arpa report generation', () => {
Expand All @@ -11,4 +12,12 @@ describe('arpa report generation', () => {
});
});

describe('audit report generation', () => {
it('generates a report', async () => {
const tenantId = 0;
const report = await withTenantId(tenantId, () => generate('http://localhost'));
assert.ok(report);
});
});

// NOTE: This file was copied from tests/server/services/generate-arpa-report.spec.js (git @ ada8bfdc98) in the arpa-reporter repo on 2022-09-23T20:05:47.735Z
Loading

0 comments on commit 0459b8a

Please sign in to comment.