From ea82e8580c48c4270a630b4b93ad678462f04b57 Mon Sep 17 00:00:00 2001 From: Dave M <64168322+replicantSocks@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:52:39 -0400 Subject: [PATCH] Move export csv to grants next (#1900) --- .../client/src/components/GrantsTableNext.vue | 9 +- packages/client/src/store/modules/grants.js | 100 ++++++++------- packages/server/__tests__/api/grants.test.js | 121 ++++++++++++++++++ packages/server/src/routes/grants.js | 113 +++++++++++++--- 4 files changed, 274 insertions(+), 69 deletions(-) diff --git a/packages/client/src/components/GrantsTableNext.vue b/packages/client/src/components/GrantsTableNext.vue index d33f9dbc4..1b2ca12f6 100644 --- a/packages/client/src/components/GrantsTableNext.vue +++ b/packages/client/src/components/GrantsTableNext.vue @@ -383,13 +383,14 @@ export default { }, exportCSV() { this.navigateToExportCSV({ + perPage: this.perPage, + currentPage: this.currentPage, orderBy: this.orderBy, orderDesc: this.orderDesc, - interestedByAgency: this.showInterested || this.showResult || this.showRejected, + showInterested: this.showInterested, + showResult: this.showResult, + showRejected: this.showRejected, assignedToAgency: this.showAssignedToAgency, - opportunityStatuses: this.parseOpportunityStatusFilters(), - opportunityCategories: this.opportunityCategoryFilters, - costSharing: this.costSharingFilter, }); }, formatMoney(value) { diff --git a/packages/client/src/store/modules/grants.js b/packages/client/src/store/modules/grants.js index d215181f2..1b2f38e82 100644 --- a/packages/client/src/store/modules/grants.js +++ b/packages/client/src/store/modules/grants.js @@ -40,6 +40,48 @@ function initialState() { }; } +function buildGrantsNextQuery({ filters, ordering, pagination }) { + /* + costSharing + eligibility + excludeKeywords + fundingType + includeKeywords + opportunityCategories + opportunityNumber + opportunityStatuses + postedWithin + reviewStatus + bill + */ + const criteria = { ...filters }; + // Validate and fix the inputs into appropriate types. + criteria.includeKeywords = criteria.includeKeywords && criteria.includeKeywords.length > 0 ? criteria.includeKeywords.split(',').map((k) => k.trim()) : null; + criteria.excludeKeywords = criteria.excludeKeywords && criteria.excludeKeywords.length > 0 ? criteria.excludeKeywords.split(',').map((k) => k.trim()) : null; + criteria.eligibility = criteria.eligibility?.map((e) => e.code); + criteria.fundingTypes = criteria.fundingTypes?.map((f) => f.code); + + const paginationQuery = Object.entries(pagination) + // filter out undefined and nulls since api expects parameters not present as undefined + // eslint-disable-next-line no-unused-vars + .filter(([key, value]) => value || typeof value === 'number') + .map(([key, value]) => `pagination[${encodeURIComponent(key)}]=${encodeURIComponent(value)}`) + .join('&'); + const orderingQuery = Object.entries(ordering) + // filter out undefined and nulls since api expects parameters not present as undefined + // eslint-disable-next-line no-unused-vars + .filter(([key, value]) => value || typeof value === 'number') + .map(([key, value]) => `ordering[${encodeURIComponent(key)}]=${encodeURIComponent(value)}`) + .join('&'); + const criteriaQuery = Object.entries(criteria) + // filter out undefined and nulls since api expects parameters not present as undefined + // eslint-disable-next-line no-unused-vars + .filter(([key, value]) => (typeof value === 'string' && value.length > 0) || typeof value === 'number' || (Array.isArray(value) && value.length > 0)) + .map(([key, value]) => `criteria[${encodeURIComponent(key)}]=${encodeURIComponent(value)}`) + .join('&'); + return { criteriaQuery, paginationQuery, orderingQuery }; +} + export default { namespaced: true, state: initialState, @@ -90,46 +132,10 @@ export default { fetchGrantsNext({ commit }, { currentPage, perPage, orderBy, orderDesc, }) { - /* - costSharing - eligibility - excludeKeywords - fundingType - includeKeywords - opportunityCategories - opportunityNumber - opportunityStatuses - postedWithin - reviewStatus - bill - */ const pagination = { currentPage, perPage }; const ordering = { orderBy, orderDesc }; - const criteria = { ...this.state.grants.searchFormFilters }; - // Validate and fix the inputs into appropriate types. - criteria.includeKeywords = criteria.includeKeywords && criteria.includeKeywords.length > 0 ? criteria.includeKeywords.split(',').map((k) => k.trim()) : null; - criteria.excludeKeywords = criteria.excludeKeywords && criteria.excludeKeywords.length > 0 ? criteria.excludeKeywords.split(',').map((k) => k.trim()) : null; - criteria.eligibility = criteria.eligibility?.map((e) => e.code); - criteria.fundingTypes = criteria.fundingTypes?.map((f) => f.code); - - const paginationQuery = Object.entries(pagination) - // filter out undefined and nulls since api expects parameters not present as undefined - // eslint-disable-next-line no-unused-vars - .filter(([key, value]) => value || typeof value === 'number') - .map(([key, value]) => `pagination[${encodeURIComponent(key)}]=${encodeURIComponent(value)}`) - .join('&'); - const orderingQuery = Object.entries(ordering) - // filter out undefined and nulls since api expects parameters not present as undefined - // eslint-disable-next-line no-unused-vars - .filter(([key, value]) => value || typeof value === 'number') - .map(([key, value]) => `ordering[${encodeURIComponent(key)}]=${encodeURIComponent(value)}`) - .join('&'); - const criteriaQuery = Object.entries(criteria) - // filter out undefined and nulls since api expects parameters not present as undefined - // eslint-disable-next-line no-unused-vars - .filter(([key, value]) => (typeof value === 'string' && value.length > 0) || typeof value === 'number' || (Array.isArray(value) && value.length > 0)) - .map(([key, value]) => `criteria[${encodeURIComponent(key)}]=${encodeURIComponent(value)}`) - .join('&'); + const filters = { ...this.state.grants.searchFormFilters }; + const { criteriaQuery, paginationQuery, orderingQuery } = buildGrantsNextQuery({ filters, ordering, pagination }); return fetchApi.get(`/api/organizations/:organizationId/grants/next?${paginationQuery}&${orderingQuery}&${criteriaQuery}`) .then((data) => commit('SET_GRANTS', data)); @@ -230,14 +236,16 @@ export default { changeEditingSearchId({ commit }, searchId) { commit('SET_EDITING_SEARCH_ID', searchId); }, - exportCSV(context, queryParams) { - const query = Object.entries(queryParams) - // filter out undefined and nulls since api expects parameters not present as undefined - // eslint-disable-next-line no-unused-vars - .filter(([key, value]) => value || typeof value === 'number') - .map(([key, value]) => `${encodeURIComponent(key)}=${encodeURIComponent(value)}`) - .join('&'); - const navUrl = fetchApi.apiURL(fetchApi.addOrganizationId(`/api/organizations/:organizationId/grants/exportCSV?${query}`)); + exportCSV({ + currentPage, perPage, orderBy, orderDesc, + }) { + const pagination = { currentPage, perPage }; + const ordering = { orderBy, orderDesc }; + const filters = { ...this.state.grants.searchFormFilters }; + const { criteriaQuery, paginationQuery, orderingQuery } = buildGrantsNextQuery({ filters, ordering, pagination }); + const navUrl = fetchApi.apiURL(fetchApi.addOrganizationId( + `/api/organizations/:organizationId/grants/exportCSVNew?${paginationQuery}&${orderingQuery}&${criteriaQuery}`, + )); window.location = navUrl; }, exportCSVRecentActivities() { diff --git a/packages/server/__tests__/api/grants.test.js b/packages/server/__tests__/api/grants.test.js index 7780c8203..e22de7d37 100644 --- a/packages/server/__tests__/api/grants.test.js +++ b/packages/server/__tests__/api/grants.test.js @@ -463,6 +463,127 @@ describe('`/api/grants` endpoint', () => { }); }); }); + context('GET /api/grants/exportCSVNew', () => { + it('produces correct column format', async () => { + // We constrain the result to a single grant that's listed in seeds/dev/ref/grants.js + const query = '?criteria[includeKeywords]=Community%20Health%20Aide Program:%20%20Tribal%20Planning'; + const response = await fetchApi(`/grants/exportCSVNew${query}`, agencies.own, fetchOptions.staff); + + expect(response.statusText).to.equal('OK'); + expect(response.headers.get('Content-Type')).to.include('text/csv'); + expect(response.headers.get('Content-Disposition')).to.include('attachment'); + + const expectedCsvHeaders = [ + 'Opportunity Number', + 'Title', + 'Viewed By', + 'Interested Agencies', + 'Status', + 'Opportunity Category', + 'Cost Sharing', + 'Award Floor', + 'Award Ceiling', + 'Posted Date', + 'Close Date', + 'Agency Code', + 'Grant Id', + 'URL', + ]; + const txt = await response.text(); + expect(txt.split('\n')[0]).to.equal(expectedCsvHeaders.join(',')); + expect(txt.split('\n')[1]).to.contain('HHS-2021-IHS-TPI-0001,Community Health Aide Program: Tribal Planning &'); + }); + + it('produces same number of rows as grid', async () => { + let response = await fetchApi(`/grants/exportCSVNew`, agencies.own, fetchOptions.staff); + expect(response.statusText).to.equal('OK'); + expect(response.headers.get('Content-Type')).to.include('text/csv'); + expect(response.headers.get('Content-Disposition')).to.include('attachment'); + const responseText = await response.text(); + const exportedRows = responseText.split(/\r?\n/); + const rowsHash = {}; + let skipFirst = true; + // eslint-disable-next-line no-restricted-syntax + for (const row of exportedRows) { + if (skipFirst) { + skipFirst = false; + } else { + const cells = row.split(','); + if (cells[0]) { + rowsHash[cells[0]] = row; + } + } + } + let pageNumber = 1; + // eslint-disable-next-line no-constant-condition + while (true) { + // eslint-disable-next-line no-await-in-loop + response = await fetchApi(`/grants/next?pagination[currentPage]=${pageNumber}&pagination[perPage]=10&ordering[orderBy]=open_date&ordering[orderDesc]=true`, agencies.own, fetchOptions.staff); + expect(response.statusText).to.equal('OK'); + // eslint-disable-next-line no-await-in-loop + const queryJson = await response.json(); + if (queryJson.data.length === 0) { + break; + } + for (let j = 0; j < queryJson.data.length; j += 1) { + const opportunityNumber = queryJson.data[j].grant_number; + if (rowsHash[opportunityNumber]) { + delete rowsHash[opportunityNumber]; + } + } + pageNumber += 1; + } + const extraRowCount = Object.keys(rowsHash).length; + if (extraRowCount > 0) { + console.log(JSON.stringify(rowsHash, null, 2)); + expect(extraRowCount).to.equal(0); + } + }); + + it('limits number of output rows', async function testExport() { + // First we insert 100 grants (in prod this limit it 10k but it is reduced in test + // via NODE_ENV=test environment variable so this test isn't so slow) + const numToInsert = 100; + const grantsToInsert = Array(numToInsert).fill(undefined).map((val, i, arr) => ({ + status: 'inbox', + grant_id: String(-(i + 1)), + grant_number: String(-(i + 1)), + agency_code: 'fake', + cost_sharing: 'No', + title: `fake grant #${i + 1}/${arr.length} for test`, + cfda_list: 'fake', + open_date: '2022-04-22', + close_date: '2022-04-22', + notes: 'auto-inserted by test', + search_terms: '[in title/desc]+', + reviewer_name: 'none', + opportunity_category: 'Discretionary', + description: 'fake grant inserted by test', + eligibility_codes: '25', + opportunity_status: 'posted', + raw_body: 'raw body', + })); + + try { + this.timeout(10000); + await knex.batchInsert(TABLES.grants, grantsToInsert); + + const response = await fetchApi(`/grants/exportCSVNew`, agencies.own, fetchOptions.staff); + expect(response.statusText).to.equal('OK'); + + const csv = await response.text(); + const lines = csv.split('\n'); + + // 10k rows + 1 header + 1 error message row + line break at EOF + expect(lines.length).to.equal(numToInsert + 3); + + const lastRow = lines[lines.length - 2]; + expect(lastRow).to.include('Error:'); + } finally { + await knex(TABLES.grants).where(knex.raw('cast(grant_id as INTEGER) < 0')).delete(); + } + }); + }); context('GET /api/grants/exportCSV', () => { it('produces correct column format', async () => { // We constrain the result to a single grant that's listed in seeds/dev/ref/grants.js diff --git a/packages/server/src/routes/grants.js b/packages/server/src/routes/grants.js index ae1fa1b73..319f41298 100755 --- a/packages/server/src/routes/grants.js +++ b/packages/server/src/routes/grants.js @@ -66,28 +66,34 @@ router.get('/', requireUser, async (req, res) => { res.json(grants); }); -router.get('/next', requireUser, async (req, res) => { - const { user } = req.session; +function criteriaToFiltersObj(criteria, agencyId) { + const filters = criteria || {}; const postedWithinOptions = { 'All Time': 0, 'One Week': 7, '30 Days': 30, '60 Days': 60, }; - const filters = req.query.criteria || {}; + + return { + reviewStatuses: filters.reviewStatus?.split(',').filter((r) => r !== 'Assigned').map((r) => r.trim()) || [], + eligibilityCodes: filters.eligibility?.split(',') || [], + includeKeywords: filters.includeKeywords?.split(',').map((k) => k.trim()) || [], + excludeKeywords: filters.excludeKeywords?.split(',').map((k) => k.trim()) || [], + opportunityNumber: filters.opportunityNumber || '', + fundingTypes: filters.fundingTypes?.split(',') || [], + opportunityStatuses: filters.opportunityStatuses?.split(',') || [], + opportunityCategories: filters.opportunityCategories?.split(',') || [], + costSharing: filters.costSharing || '', + agencyCode: filters.agency || '', + postedWithinDays: postedWithinOptions[filters.postedWithin] || 0, + assignedToAgencyId: filters.reviewStatus?.includes('Assigned') ? agencyId : null, + bill: filters.bill || null, + }; +} + +router.get('/next', requireUser, async (req, res) => { + const { user } = req.session; + const grants = await db.getGrantsNew( - { - reviewStatuses: filters.reviewStatus?.split(',').filter((r) => r !== 'Assigned').map((r) => r.trim()) || [], - eligibilityCodes: filters.eligibility?.split(',') || [], - includeKeywords: filters.includeKeywords?.split(',').map((k) => k.trim()) || [], - excludeKeywords: filters.excludeKeywords?.split(',').map((k) => k.trim()) || [], - opportunityNumber: filters.opportunityNumber || '', - fundingTypes: filters.fundingTypes?.split(',') || [], - opportunityStatuses: filters.opportunityStatuses?.split(',') || [], - opportunityCategories: filters.opportunityCategories?.split(',') || [], - costSharing: filters.costSharing || '', - agencyCode: filters.agency || '', - postedWithinDays: postedWithinOptions[filters.postedWithin] || 0, - assignedToAgencyId: filters.reviewStatus?.includes('Assigned') ? user.agency_id : null, - bill: filters.bill || null, - }, + criteriaToFiltersObj(req.query.criteria, user.agency_id), await db.buildPaginationParams(req.query.pagination), await db.buildOrderingParams(req.query.ordering), user.tenant_id, @@ -119,7 +125,76 @@ router.get('/exportCSVNext', requireUser, async (req, res) => { // For API tests, reduce the limit to 100 -- this is so we can test the logic around the limit // without the test having to insert 10k rows, which slows down the test. -const MAX_CSV_EXPORT_ROWS = process.env.NODE_ENV !== 'test' ? 10000 : 100; +const MAX_CSV_EXPORT_ROWS = process.env.NODE_ENV !== 'test' ? 300 : 100; + +router.get('/exportCSVNew', requireUser, async (req, res) => { + const { user } = req.session; + + const { data, pagination } = await db.getGrantsNew( + criteriaToFiltersObj(req.query.criteria, user.agency_id), + await db.buildPaginationParams({ + currentPage: 1, + perPage: MAX_CSV_EXPORT_ROWS, + }), + await db.buildOrderingParams(req.query.ordering), + user.tenant_id, + user.agency_id, + ); + + // Generate CSV + const formattedData = data.map((grant) => ({ + ...grant, + interested_agencies: grant.interested_agencies + .map((v) => v.agency_abbreviation) + .join(', '), + viewed_by: grant.viewed_by_agencies + .map((v) => v.agency_abbreviation) + .join(', '), + open_date: new Date(grant.open_date).toLocaleDateString('en-US', { timeZone: 'UTC' }), + close_date: new Date(grant.close_date).toLocaleDateString('en-US', { timeZone: 'UTC' }), + award_floor: getAwardFloor(grant), + url: `https://www.grants.gov/web/grants/view-opportunity.html?oppId=${grant.grant_id}`, + })); + + if (data.length === 0) { + // If there are 0 rows, csv-stringify won't even emit the header, resulting in a totally + // empty file, which is confusing. This adds a single empty row below the header. + formattedData.push({}); + } else if (pagination.total > data.length) { + formattedData.push({ + title: `Error: only ${MAX_CSV_EXPORT_ROWS} rows supported for CSV export, but there ` + + `are ${pagination.total} total.`, + }); + } + + const csv = csvStringify(formattedData, { + header: true, + columns: [ + { key: 'grant_number', header: 'Opportunity Number' }, + { key: 'title', header: 'Title' }, + { key: 'viewed_by', header: 'Viewed By' }, + { key: 'interested_agencies', header: 'Interested Agencies' }, + { key: 'opportunity_status', header: 'Status' }, + { key: 'opportunity_category', header: 'Opportunity Category' }, + { key: 'cost_sharing', header: 'Cost Sharing' }, + { key: 'award_floor', header: 'Award Floor' }, + { key: 'award_ceiling', header: 'Award Ceiling' }, + { key: 'open_date', header: 'Posted Date' }, + { key: 'close_date', header: 'Close Date' }, + { key: 'agency_code', header: 'Agency Code' }, + { key: 'grant_id', header: 'Grant Id' }, + { key: 'url', header: 'URL' }, + ], + }); + + // Send to client as a downloadable file. + const filename = 'grants.csv'; + res.setHeader('Content-Disposition', `attachment; filename=${filename}`); + res.setHeader('Content-Type', 'text/csv'); + res.setHeader('Content-Length', csv.length); + res.send(csv); +}); + router.get('/exportCSV', requireUser, async (req, res) => { const { selectedAgency, user } = req.session; // First load the grants. This logic is intentionally identical to the endpoint above that