Merge pull request #2565 from GSA-TTS/main

[2023-10-20] main -> prod

backend/.profile

@@ -28,7 +28,7 @@ export NEW_RELIC_ENVIRONMENT="$(echo "$VCAP_APPLICATION" | jq -r .space_name)"
 export NEW_RELIC_LOG=stdout
 
 # Logging level, (critical, error, warning, info and debug). Default to info
-export NEW_RELIC_LOG_LEVEL=debug
+export NEW_RELIC_LOG_LEVEL=error
 
 # https://docs.newrelic.com/docs/security/security-privacy/compliance/fedramp-compliant-endpoints/
 export NEW_RELIC_HOST="gov-collector.newrelic.com"

backend/audit/fixtures/single_audit_checklist.py

@@ -210,8 +210,6 @@ def _load_single_audit_checklists_for_user(user):
         if sac is None:
             # need to make this object
             sac = _create_sac(user, auditee_name)
-        if "post_create_callable" in item_info:
-            item_info["post_create_callable"](sac, user)
 
 
 def load_single_audit_checklists():

backend/audit/fixtures/workbooks/should_pass/100010-22/test-array-100010.json

@@ -1,31 +1,31 @@
 [
   {
     "endpoint": "additional_eins",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "additional_ueis",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "corrective_action_plans",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "federal_awards",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [
       {
         "fields": [
@@ -353,21 +353,21 @@
   },
   {
     "endpoint": "findings_text",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "findings",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {}
   },
   {
     "endpoint": "notes_to_sefa",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "accounting_policies": "Expenditures reported on the Schedule are reported on the accrual basis of accounting.Such expenditures are recognized following, as applicable, either the cost principles in OMB Circular A-87, Cost Principles for State, Local and Indian Tribal Governments, or the cost principles contained in the Uniform Guidance, wherein certain types of expenditures are not allowable or are limited as to reimbursement.",
@@ -378,7 +378,7 @@
   },
   {
     "endpoint": "secondary_auditor",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"

backend/audit/intakelib/checks/check_loan_guarantee.py

@@ -32,7 +32,7 @@ def loan_guarantee(ir):
             errors.append(
                 build_cell_error_tuple(
                     ir,
-                    get_range_by_name(ir, "loan_balance_at_audit_period_end"),
+                    get_range_by_name(ir, "is_guaranteed"),
                     index,
                     get_message("check_loan_guarantee_not_empty"),
                 )

backend/audit/intakelib/mapping_notes_to_sefa.py

@@ -24,7 +24,6 @@
 from .checks import run_all_general_checks, run_all_notes_to_sefa_checks
 from .transforms import run_all_notes_to_sefa_transforms
 
-
 logger = logging.getLogger(__name__)
 
 

backend/audit/intakelib/transforms/runners.py

@@ -14,6 +14,9 @@
 )
 
 from .xform_eins_need_to_be_strings import eins_need_to_be_strings
+from .xform_rename_additional_notes_sheet import (
+    rename_additional_notes_sheet_to_form_sheet,
+)
 
 from .xform_all_alns_need_to_be_strings import all_alns_need_to_be_strings
 from .xform_all_passthrough_id_need_to_be_strings import (
@@ -46,6 +49,7 @@ def run_all_federal_awards_transforms(ir):
 
 notes_to_sefa_transforms = general_transforms + [
     trim_null_from_content_fields_in_notes_to_sefa,
+    rename_additional_notes_sheet_to_form_sheet,
     insert_sequence_nums_into_notes_to_sefa,
 ]
 

backend/audit/intakelib/transforms/xform_insert_sequence_nums_into_notes_to_sefa.py

@@ -10,5 +10,5 @@
 def insert_sequence_nums_into_notes_to_sefa(ir):
     contains_range = get_range_by_name(ir, "contains_chart_or_table")
     new_values = list(map(lambda v: v + 1, range(len(contains_range["values"]))))
-    new_ir = insert_new_range(ir, "AdditionalNotes", "seq_number", "D", 2, new_values)
+    new_ir = insert_new_range(ir, "Form", "seq_number", "D", 2, new_values)
     return new_ir

backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py

@@ -0,0 +1,15 @@
+import logging
+from copy import deepcopy
+
+logger = logging.getLogger(__name__)
+
+
+# This transform is needed for backwards compatibility with workbook templates 1.0.0 and 1.0.1
+def rename_additional_notes_sheet_to_form_sheet(ir):
+    new_ir = deepcopy(ir)
+
+    for sheet in new_ir:
+        if sheet["name"] == "AdditionalNotes":
+            sheet["name"] = "Form"
+
+    return new_ir

backend/audit/test_commands.py

@@ -7,8 +7,7 @@
 from django.test import TestCase
 from unittest.mock import patch
 
-from audit.models import SingleAuditChecklist, ExcelFile
-from audit.fixtures.excel import FORM_SECTIONS
+from audit.models import SingleAuditChecklist
 
 
 class MockHttpResponse:
@@ -47,24 +46,3 @@ def test_load_fixtures(self, mock_scan_file):
 
         # restore the logging override
         logging.disable(logging.ERROR)
-
-    @patch("audit.validators._scan_file")
-    def test_load_fixtures_federal_awards(self, mock_scan_file):
-        """load_fixtures command makes a SAC with federal awards."""
-        # make sure we have at least one user
-        User = get_user_model()
-        user, _ = User.objects.get_or_create(username="test_at_least_one")
-
-        # mock the call to the external AV service
-        mock_scan_file.return_value = MockHttpResponse(200, "clean!")
-
-        call_command("load_fixtures")
-
-        # should be a federal awards ExcelFile for this user
-        files = ExcelFile.objects.filter(
-            user=user, form_section=FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED
-        )
-        self.assertTrue(files)
-
-        # and the associated SAC has `federal_awards` data
-        self.assertTrue(files.first().sac.federal_awards)

backend/cypress/support/full-submission.js

@@ -155,7 +155,7 @@ export function testFullSubmission(isTribal, isPublic) {
     // The report ID should be found in the Completed Audits table
     cy.get('.usa-table').contains(
       'caption',
-      'The audits listed below have been submitted to the FAC for processing and may not be edited.',
+      /audits are complete/
     ).siblings().contains('td', reportId);
 
     // The Report should not be in the dissemination table

backend/data_fixtures/audit/test_data_entries/simple-cases.json

@@ -281,7 +281,8 @@
           {
             "note_title": "Note one",
             "note_content": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. \nVestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Phasellus nec tortor ut ligula sollicitudin euismod.",
-            "contains_chart_or_table": "Y"
+            "contains_chart_or_table": "Y",
+            "seq_number":1
           }
         ]
       }

backend/dissemination/management/commands/end_to_end_workbook_test.py

@@ -1,8 +1,6 @@
-from django.apps import apps
 from django.core.management.base import BaseCommand
 from users.models import User
 import argparse
-import datetime
 import logging
 import sys
 import math
@@ -11,6 +9,7 @@
 import jwt
 import requests
 from pprint import pprint
+from datetime import datetime
 
 from dissemination.workbooklib.workbook_creation import (
     sections,
@@ -44,21 +43,13 @@
 pw.setLevel(logging.INFO)
 
 
-def step_through_certifications(sac, SAC):
-    sac.transition_name.append(SAC.STATUS.SUBMITTED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.READY_FOR_CERTIFICATION)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.AUDITOR_CERTIFIED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.AUDITEE_CERTIFIED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.CERTIFIED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_date.append(SAC.STATUS.SUBMITTED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_date.append(SAC.STATUS.DISSEMINATED)
-    sac.transition_date.append(datetime.datetime.today())
+def step_through_certifications(sac):
+    sac.transition_to_ready_for_certification()
+    sac.transition_to_auditor_certified()
+    sac.transition_to_auditee_certified()
+    sac.transition_to_submitted()
+    sac.transition_to_disseminated()
+    sac.save()
 
 
 def disseminate(sac, year):
@@ -87,7 +78,7 @@ def create_payload(api_url, role="api_fac_gov"):
     payload = {
         # PostgREST only cares about the role.
         "role": role,
-        "created": datetime.datetime.today().isoformat(),
+        "created": datetime.today().isoformat(),
     }
     return payload
 
@@ -173,6 +164,7 @@ def api_check(json_test_tables):
             equality_results = []
             for field_ndx, f in enumerate(row["fields"]):
                 # logger.info(f"Checking /{endpoint} {report_id} {f}")
+                # logger.info(f"{get_api_values(endpoint, report_id, f)}")
                 api_values = get_api_values(endpoint, report_id, f)
                 this_api_value = api_values[row_ndx]
                 this_field_value = row["values"][field_ndx]
@@ -195,7 +187,7 @@ def api_check(json_test_tables):
 
 def generate_workbooks(user, email, dbkey, year):
     entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format(
-        dbkey=dbkey, year=year, date=datetime.datetime.now()
+        dbkey=dbkey, year=year, date=datetime.now()
     )
     sac = setup_sac(user, entity_id, dbkey)
     if sac.general_information["audit_type"] == "alternative-compliance-engagement":
@@ -208,8 +200,7 @@ def generate_workbooks(user, email, dbkey, year):
             (_, json, _) = loader(fun, section)
             json_test_tables.append(json)
         _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf")
-        SingleAuditChecklist = apps.get_model("audit.SingleAuditChecklist")
-        step_through_certifications(sac, SingleAuditChecklist)
+        step_through_certifications(sac)
 
         # shaped_sac = sac_validation_shape(sac)
         # result = submission_progress_check(shaped_sac, sar=None, crossval=False)

backend/dissemination/workbooklib/federal_awards.py

@@ -126,11 +126,11 @@ def _fix_pfixes(cfdas):
     extensions = map(lambda v: ((v.cfda).split(".")[1])[:3].upper(), cfdas)
     extensions = map(
         lambda v: v
-        if re.search("^(RD|[0-9]{3}[A-Za-z]{0,1}|U[0-9]{2})$", v)
+        if re.search("^(RD|RD[0-9]|[0-9]{3}[A-Za-z]{0,1}|U[0-9]{2})$", v)
         else "000",
         extensions,
     )
-    return (prefixes, extensions)
+    return (prefixes, extensions, map(lambda v: v.cfda, cfdas))
 
 
 def _fix_passthroughs(Cfda, Passthrough, cfdas, dbkey):
@@ -213,10 +213,13 @@ def generate_federal_awards(dbkey, year, outfile):
     addls = _fix_addl_award_identification(Cfda, cfdas, dbkey)
     set_range(wb, "additional_award_identification", addls)
 
-    (prefixes, extensions) = _fix_pfixes(cfdas)
+    (prefixes, extensions, full_cfdas) = _fix_pfixes(cfdas)
     set_range(wb, "federal_agency_prefix", prefixes)
     set_range(wb, "three_digit_extension", extensions)
 
+    # We need a `cfda_key` as a magic column for the summation logic to work/be checked.
+    set_range(wb, "cfda_key", full_cfdas, conversion_fun=str)
+
     (passthrough_names, passthrough_ids) = _fix_passthroughs(
         Cfda, Passthrough, cfdas, dbkey
     )

backend/dissemination/workbooklib/sac_creation.py

@@ -259,6 +259,25 @@ def _create_test_sac(user, auditee_name, dbkey):
         email="[email protected]",  # user.email,
         role="certifying_auditor_contact",
     )
+
+    sac.auditee_certification = {}
+    sac.auditee_certification["auditee_signature"] = {}
+    sac.auditee_certification["auditee_signature"][
+        "auditee_name"
+    ] = "Bob the Auditee Name"
+    sac.auditee_certification["auditee_signature"][
+        "auditee_title"
+    ] = "Bob the Auditee Signature"
+
+    sac.auditor_certification = {}
+    sac.auditor_certification["auditor_signature"] = {}
+    sac.auditor_certification["auditor_signature"][
+        "auditor_name"
+    ] = "Alice the Auditor Name"
+    sac.auditor_certification["auditor_signature"][
+        "auditor_title"
+    ] = "Alice the Auditor Signature"
+
     sac.data_source = "TSTDAT"
     sac.save()
 

backend/schemas/output/excel/json/additional-eins-workbook.json

@@ -31,7 +31,7 @@
          "single_cells": [
             {
                "format": "text",
-               "formula": "=\"1.0.1\"",
+               "formula": "=\"1.0.2\"",
                "help": {
                   "link": "https://fac.gov/documentation/validation/#plain_text",
                   "text": "Only plain text is allowed, no emoji, formatting, or other special additions"

backend/schemas/output/excel/json/additional-ueis-workbook.json

@@ -31,7 +31,7 @@
          "single_cells": [
             {
                "format": "text",
-               "formula": "=\"1.0.1\"",
+               "formula": "=\"1.0.2\"",
                "help": {
                   "link": "https://fac.gov/documentation/validation/#plain_text",
                   "text": "Only plain text is allowed, no emoji, formatting, or other special additions"

backend/schemas/output/excel/json/audit-findings-text-workbook.json

@@ -31,7 +31,7 @@
          "single_cells": [
             {
                "format": "text",
-               "formula": "=\"1.0.1\"",
+               "formula": "=\"1.0.2\"",
                "help": {
                   "link": "https://fac.gov/documentation/validation/#plain_text",
                   "text": "Only plain text is allowed, no emoji, formatting, or other special additions"
@@ -141,7 +141,8 @@
                },
                "width": 36
             }
-         ]
+         ],
+         "row_height": 36
       }
    ],
    "title_row": 1