Merge branch '_staging' into 1894-audit-report-enhancements

.github/workflows/build-and-deploy.yml

@@ -197,8 +197,7 @@ jobs:
     if: |
       always() &&
       needs.build_api.result == 'success' &&
-      (needs.deploy_terraform.result == 'success' || needs.deploy_terraform.result == 'skipped') &&
-      needs.select_target_environment.outputs.selected == 'staging'
+      (needs.deploy_terraform.result == 'success' || needs.deploy_terraform.result == 'skipped')
     environment: ${{ needs.select_target_environment.outputs.selected }}
     steps:
       - name: Configure AWS Credentials

packages/client/src/arpa_reporter/views/Home.vue

@@ -111,7 +111,7 @@ export default {
       this.sending = true;
 
       try {
-        const result = await getJson('/api/audit_report?async=true');
+        const result = await getJson('/api/audit_report?queue=true');
 
         if (result.error) {
           this.alert = {

packages/client/src/helpers/constants.js

@@ -1,41 +1,37 @@
 const billOptions = [
   'All Bills',
-  'Department of Justice',
+  'Bipartisan Safer Communities Act',
+  'Consumer Product Safety Commission',
+  'Coronavirus',
+  'Corporation for National and Community Service',
+  'Denali Commission',
+  'Department of Agriculture',
+  'Department of Commerce',
+  'Department of Defense',
+  'Department of Education',
   'Department of Energy',
-  'Department of the Treasury',
-  '12.005',
   'Department of Health and Human Services',
-  'Department of Defense',
-  'Denali Commission',
-  'Bipartisan Safer Communities Act',
-  'National Aeronautics & Space Administration',
-  'Department of Labor',
+  'Department of Homeland Security',
   'Department of Housing and Urban Development',
-  'Department of Transportation',
-  'Infrastructure Investment and jobs Act',
-  'Corporation for National and Community Service',
+  'Department of Justice',
+  'Department of Labor',
   'Department of the Interior',
-  'National Foundation on the Arts and the Humanities',
-  'Department of Homeland Security',
-  'National Archives & Records Administration',
+  'Department of the Treasury',
+  'Department of Transportation',
+  'Department of Veterans Affairs',
   'Environmental Protection Agency',
+  'Inflation Reduction Act',
   'Infrastructure Investment and Jobs Act',
-  'Coronavirus',
-  'Department of Agriculture',
-  'Consumer Product Safety Commission',
-  'Infrastructure Investment and Jobs Act (IIJA)',
-  'Small Business Administration',
-  'Office of National Drug Control Policy',
-  'Department of Education',
+  'Library of Congress',
+  'National Aeronautics & Space Administration',
+  'National Archives & Records Administration',
   'National Council on Disability',
-  'Nuclear Regulatory Commission',
-  'Social Security Administration',
-  'Department of Veterans Affairs',
+  'National Foundation on the Arts and the Humanities',
   'National Science Foundation',
-  'Library of Congress',
+  'Nuclear Regulatory Commission',
   'Office of National Drug Control Policy',
-  'Department of Commerce',
-  'Inflation Reduction Act',
+  'Small Business Administration',
+  'Social Security Administration',
 ];
 
 module.exports = {

packages/server/__tests__/db/db.test.js

@@ -5,6 +5,7 @@ const db = require('../../src/db');
 const { TABLES } = require('../../src/db/constants');
 const fixtures = require('./seeds/fixtures');
 const emailConstants = require('../../src/lib/email/constants');
+const keywordMigrationScript = require('../../src/db/saved_search_migration');
 
 const BASIC_SEARCH_CRITERIA = JSON.stringify({
     includeKeywords: 'Grant',
@@ -27,6 +28,102 @@ describe('db', () => {
     after(async () => {
         await db.knex.destroy();
     });
+    context('migrate keywords to saved search', () => {
+        it('migrates keywords to saved search dry-run', async () => {
+            const fakeLog = sinon.fake.returns('foo');
+            keywordMigrationScript.log = fakeLog;
+            process.argv.push('--dry-run');
+            await keywordMigrationScript.migrate_keywords_to_saved_search();
+            process.argv.pop();
+
+            const expectedLogOutput = [
+                'DRY RUN :: Begin migrating legacy agency criteria to saved searches',
+                'DRY RUN :: Migrating agency criteria for agency 0',
+                'DRY RUN :: No agency criteria to migrate for agency 1',
+                'DRY RUN :: Migrating agency criteria for agency 4',
+                'DRY RUN :: Migrating agency criteria for users 1,2 belonging to agency 0',
+                'DRY RUN :: No agency criteria to migrate for users 3 belonging to agency 1',
+                'DRY RUN :: No users to migrate for agency 4',
+                'DRY RUN :: Would have inserted approximately 2 saved searches. Note: there may be duplicates.',
+                'DRY RUN :: Done migrating legacy agency criteria to saved searches',
+            ];
+            const actualLogOutput = [];
+            fakeLog.getCalls().forEach((call) => { actualLogOutput.push(call.firstArg); });
+            expect(actualLogOutput.join(',\n')).to.equal(expectedLogOutput.join(',\n'));
+        });
+        it('migrations keywords to saved search no duplicates', async () => {
+            const fakeLog = sinon.fake.returns('foo');
+            keywordMigrationScript.log = fakeLog;
+            await keywordMigrationScript.migrate_keywords_to_saved_search();
+            const rows = await knex('grants_saved_searches').where('name', 'Legacy - Saved Search');
+            await knex('grants_saved_searches')
+                .where('name', 'Legacy - Saved Search')
+                .delete();
+
+            const expectedLogOutput = [
+                'Begin migrating legacy agency criteria to saved searches',
+                'Migrating agency criteria for agency 0',
+                'No agency criteria to migrate for agency 1',
+                'Migrating agency criteria for agency 4',
+                'Migrating agency criteria for users 1,2 belonging to agency 0',
+                'No agency criteria to migrate for users 3 belonging to agency 1',
+                'No users to migrate for agency 4',
+                'Inserted 2 saved searches',
+                'Done migrating legacy agency criteria to saved searches',
+            ];
+            const actualLogOutput = [];
+            fakeLog.getCalls().forEach((call) => { actualLogOutput.push(call.firstArg); });
+            expect(actualLogOutput.join(',\n')).to.equal(expectedLogOutput.join(',\n'));
+            rows.forEach((row) => {
+                expect(db.validateSearchFilters(db.formatSearchCriteriaToQueryFilters(row.criteria))).to.have.lengthOf(0);
+            });
+        });
+        it('migrates keywords to saved search ignores duplicates', async () => {
+            const fakeLog = sinon.fake.returns('foo');
+            keywordMigrationScript.log = fakeLog;
+            await knex('grants_saved_searches')
+                .insert({
+                    created_by: 1,
+                    criteria: {
+                        opportunityStatuses: ['posted'],
+                        fundingTypes: null,
+                        agency: null,
+                        bill: null,
+                        costSharing: null,
+                        opportunityCategories: [],
+                        postedWithin: [],
+                        includeKeywords: 'Covid',
+                        excludeKeywords: 'Climate',
+                        eligibility: [],
+                    },
+                    name: 'Legacy - Saved Search',
+                })
+                .returning('id');
+            await keywordMigrationScript.migrate_keywords_to_saved_search();
+            const rows = await knex('grants_saved_searches').where('name', 'Legacy - Saved Search');
+            await knex('grants_saved_searches')
+                .where('name', 'Legacy - Saved Search')
+                .delete();
+
+            const expectedLogOutput = [
+                'Begin migrating legacy agency criteria to saved searches',
+                'Migrating agency criteria for agency 0',
+                'No agency criteria to migrate for agency 1',
+                'Migrating agency criteria for agency 4',
+                'Migrating agency criteria for users 1,2 belonging to agency 0',
+                'No agency criteria to migrate for users 3 belonging to agency 1',
+                'No users to migrate for agency 4',
+                'Inserted 1 saved searches', // This would have been 2 if not for the duplication mechanism.
+                'Done migrating legacy agency criteria to saved searches',
+            ];
+            const actualLogOutput = [];
+            fakeLog.getCalls().forEach((call) => { actualLogOutput.push(call.firstArg); });
+            expect(actualLogOutput.join(',\n')).to.equal(expectedLogOutput.join(',\n'));
+            rows.forEach((row) => {
+                expect(db.validateSearchFilters(db.formatSearchCriteriaToQueryFilters(row.criteria))).to.have.lengthOf(0);
+            });
+        });
+    });
     context('Validate Search Filters', () => {
         it('throws an error when non-existent option is passed', async () => {
             const badFilters = {

packages/server/package.json

@@ -14,6 +14,7 @@
     "consume-grants": "node src/scripts/consumeGrantModifications.js",
     "coverage": "nyc yarn test",
     "create_tenant": "node src/db/tenant_creation.js",
+    "migrate_saved_searches": "node src/db/saved_search_migration.js",
     "db:migrate": "knex migrate:latest",
     "db:rollback": "knex migrate:rollback",
     "db:seed": "yarn db:migrate && knex seed:run",

packages/server/src/arpa_reporter/db/reporting-periods.js

@@ -35,16 +35,13 @@ function baseQuery(trns) {
         .leftJoin('users', 'reporting_periods.certified_by', 'users.id');
 }
 
-async function getAllReportingPeriods(trns = knex) {
-    const tenantId = useTenantId();
+async function getAllReportingPeriods(trns = knex, tenantId = useTenantId()) {
     return baseQuery(trns).where('reporting_periods.tenant_id', tenantId).orderBy('end_date', 'desc');
 }
 
 /* getReportingPeriod() returns a record from the reporting_periods table.
   */
-async function getReportingPeriod(period_id = undefined, trns = knex) {
-    const tenantId = useTenantId();
-
+async function getReportingPeriod(period_id = undefined, trns = knex, tenantId = useTenantId()) {
     if (period_id && Number(period_id)) {
         return baseQuery(trns)
             .where('reporting_periods.tenant_id', tenantId)
@@ -63,8 +60,8 @@ async function getReportingPeriod(period_id = undefined, trns = knex) {
 /*  getPeriodID() returns the argument unchanged unless it is falsy, in which
   case it returns the current reporting period ID.
   */
-async function getReportingPeriodID(periodID) {
-    return Number(periodID) || getCurrentReportingPeriodID();
+async function getReportingPeriodID(periodID, tenantId = useTenantId()) {
+    return Number(periodID) || getCurrentReportingPeriodID(undefined, tenantId);
 }
 
 /**
@@ -73,24 +70,22 @@ async function getReportingPeriodID(periodID) {
  *
  * @returns The matching reporting periods, sorted from oldest to newest by date
  */
-async function getPreviousReportingPeriods(period_id, trns = knex) {
-    const currentReportingPeriod = await getReportingPeriod(period_id, trns);
-    const allReportingPeriods = await getAllReportingPeriods(trns);
+async function getPreviousReportingPeriods(period_id, trns = knex, tenantId) {
+    const currentReportingPeriod = await getReportingPeriod(period_id, trns, tenantId);
+    const allReportingPeriods = await getAllReportingPeriods(trns, tenantId);
     const reportingPeriods = allReportingPeriods.filter(
         (period) => new Date(period.end_date) <= new Date(currentReportingPeriod.end_date),
     );
     reportingPeriods.sort((a, b) => new Date(a.end_date) - new Date(b.end_date));
     return reportingPeriods;
 }
 
-async function closeReportingPeriod(period, trns = knex) {
-    const { user } = useRequest().session;
-    const tenantId = useTenantId();
+async function closeReportingPeriod(period, trns = knex, user = useRequest().session.user) {
     if (user.tenant_id !== period.tenant_id) {
         throw new Error('user cannot close reporting period of a different tenant');
     }
 
-    const currentPeriodID = await getCurrentReportingPeriodID(trns);
+    const currentPeriodID = await getCurrentReportingPeriodID(trns, user.tenant_id);
 
     if (period.id !== currentPeriodID) {
         throw new Error(
@@ -105,7 +100,7 @@ async function closeReportingPeriod(period, trns = knex) {
     }
 
     const prior = await trns('reporting_periods')
-        .where('tenant_id', tenantId)
+        .where('tenant_id', user.tenant_id)
         .where('start_date', '<', period.start_date)
         .orderBy('start_date', 'desc')
         .limit(1)
@@ -134,18 +129,16 @@ async function closeReportingPeriod(period, trns = knex) {
         });
 
     const next = await trns('reporting_periods')
-        .where('tenant_id', tenantId)
+        .where('tenant_id', user.tenant_id)
         .where('start_date', '>', period.start_date)
         .orderBy('start_date', 'asc')
         .limit(1)
         .then((rows) => rows[0]);
 
-    await setCurrentReportingPeriod(next.id, trns);
+    await setCurrentReportingPeriod(next.id, trns, user.tenant_id);
 }
 
-async function createReportingPeriod(reportingPeriod, trns = knex) {
-    const tenantId = useTenantId();
-
+async function createReportingPeriod(reportingPeriod, trns = knex, tenantId = useTenantId()) {
     return trns
         .insert({ ...reportingPeriod, tenant_id: tenantId })
         .into('reporting_periods')

packages/server/src/arpa_reporter/db/settings.js

@@ -2,27 +2,22 @@ const knex = require('../../db/connection');
 const { requiredArgument } = require('../lib/preconditions');
 const { useTenantId } = require('../use-request');
 
-function setCurrentReportingPeriod(id, trns = knex) {
-    const tenantId = useTenantId();
+function setCurrentReportingPeriod(id, trns = knex, tenantId = useTenantId()) {
     requiredArgument(id, 'must specify id in setCurrentReportingPeriod');
 
     return trns('application_settings')
         .where('tenant_id', tenantId)
         .update('current_reporting_period_id', id);
 }
 
-async function getCurrentReportingPeriodID(trns = knex) {
-    const tenantId = useTenantId();
-
+async function getCurrentReportingPeriodID(trns = knex, tenantId = useTenantId()) {
     return trns('application_settings')
         .select('*')
         .where('tenant_id', tenantId)
         .then((r) => r[0].current_reporting_period_id);
 }
 
-async function applicationSettings(tenantId = undefined, trns = knex) {
-    tenantId = tenantId || useTenantId();
-
+async function applicationSettings(tenantId = useTenantId(), trns = knex) {
     return trns('application_settings')
         .join(
             'reporting_periods',

packages/server/src/arpa_reporter/db/uploads.js

@@ -14,10 +14,9 @@ function baseQuery(trns) {
         .select('uploads.*', 'users.email AS created_by', 'agencies.code AS agency_code');
 }
 
-async function uploadsInPeriod(periodId, trns = knex) {
-    const tenantId = useTenantId();
+async function uploadsInPeriod(periodId, trns = knex, tenantId = useTenantId()) {
     if (periodId === undefined) {
-        periodId = await getCurrentReportingPeriodID(trns);
+        periodId = await getCurrentReportingPeriodID(trns, tenantId);
     }
 
     return baseQuery(trns)
@@ -45,8 +44,7 @@ function getUpload(id, trns = knex) {
         .then((r) => r[0]);
 }
 
-function usedForTreasuryExport(periodId, tenantId = undefined, trns = knex) {
-    tenantId = tenantId || useTenantId();
+function usedForTreasuryExport(periodId, tenantId = useTenantId(), trns = knex) {
     requiredArgument(periodId, 'periodId must be specified in validForReportingPeriod');
 
     return baseQuery(trns)
@@ -80,17 +78,14 @@ function usedForTreasuryExport(periodId, tenantId = undefined, trns = knex) {
       agency_id: 3,
     }
     */
-function getUploadSummaries(period_id, trns = knex) {
-    const tenantId = useTenantId();
+function getUploadSummaries(period_id, trns = knex, tenantId = useTenantId()) {
     // console.log(`period_id is ${period_id}`)
     return trns('uploads')
         .select('*')
         .where({ reporting_period_id: period_id, tenant_id: tenantId });
 }
 
-async function createUpload(upload, trns = knex) {
-    const tenantId = useTenantId();
-
+async function createUpload(upload, trns = knex, tenantId = useTenantId()) {
     const inserted = await trns('uploads')
         .insert({ ...upload, tenant_id: tenantId })
         .returning('*')
@@ -111,11 +106,9 @@ async function setEcCode(uploadId, ecCode, trns = knex) {
         .update({ ec_code: ecCode });
 }
 
-async function getPeriodUploadIDs(period_id, trns = knex) {
-    const tenantId = useTenantId();
-
+async function getPeriodUploadIDs(period_id, trns = knex, tenantId = useTenantId()) {
     if (!period_id) {
-        period_id = await getCurrentReportingPeriodID(trns);
+        period_id = await getCurrentReportingPeriodID(trns, tenantId);
     }
     let rv;
     try {