Jadudm fix workbook generator for new checks (#2563)

* Fixes logic in check_federal_program_total

The check was being applied against the federal_program_total line.

However, it was summing all expenditures and expecting to find the sum
of everything in `amount_expended` to appear in `federal_program_total`.

What the check wanted to do was to compare against
`total_federal_expenditures.` I suspect this is because I did not ticket
the issue clearly.

This makes the check do the right thing.

* Fixes my fix.

I was confused.

I forgot that an outcome from earlier was that we needed to fix the
workbook generator.

Therefore, I confused myself on the check_federal_program_total code.

I reverted my adventure there (leaving it as-is from HSMD's work), and
instead fixed the workbook generator to now create/insert the `cfda_key`
column.

I regenerated 100010 to verify that this now works. And, the end-to-end
code now works as well (again), because it does the right thing.

test_commands is a casualty. It is testing a fixture command that is
aging, and I think that end-to-end and the workbook generator will
replace it.

* That was too easy.

I spent too much time with date formats.

* Linting.

* Before main sync

* Addes auditor cert, fixes E2E regression

The text on the table changed, so the regression had to change. I've
switched to a regex instead of the full contents of the table caption.

This also makes the E2E workbook test code actually move audits into the
"Accepted" table.

This preps the E2E code for use in actuall regression testing.

* Linting.

* Apply suggestions from code review

Accepting cleanup suggestions.

Co-authored-by: Phil Dominguez <[email protected]>

* Removing a test.

---------

Co-authored-by: Phil Dominguez <[email protected]>

backend/audit/fixtures/single_audit_checklist.py

@@ -210,8 +210,6 @@ def _load_single_audit_checklists_for_user(user):
         if sac is None:
             # need to make this object
             sac = _create_sac(user, auditee_name)
-        if "post_create_callable" in item_info:
-            item_info["post_create_callable"](sac, user)
 
 
 def load_single_audit_checklists():

backend/audit/fixtures/workbooks/should_pass/100010-22/test-array-100010.json

@@ -1,31 +1,31 @@
 [
   {
     "endpoint": "additional_eins",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "additional_ueis",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "corrective_action_plans",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "federal_awards",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [
       {
         "fields": [
@@ -353,21 +353,21 @@
   },
   {
     "endpoint": "findings_text",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"
     }
   },
   {
     "endpoint": "findings",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {}
   },
   {
     "endpoint": "notes_to_sefa",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "accounting_policies": "Expenditures reported on the Schedule are reported on the accrual basis of accounting.Such expenditures are recognized following, as applicable, either the cost principles in OMB Circular A-87, Cost Principles for State, Local and Indian Tribal Governments, or the cost principles contained in the Uniform Guidance, wherein certain types of expenditures are not allowable or are limited as to reimbursement.",
@@ -378,7 +378,7 @@
   },
   {
     "endpoint": "secondary_auditor",
-    "report_id": "2022TEST000100010",
+    "report_id": "2022-06-TSTDAT-0000100010",
     "rows": [],
     "singletons": {
       "auditee_uei": "VBF7RJTP7513"

backend/audit/test_commands.py

@@ -7,8 +7,7 @@
 from django.test import TestCase
 from unittest.mock import patch
 
-from audit.models import SingleAuditChecklist, ExcelFile
-from audit.fixtures.excel import FORM_SECTIONS
+from audit.models import SingleAuditChecklist
 
 
 class MockHttpResponse:
@@ -47,24 +46,3 @@ def test_load_fixtures(self, mock_scan_file):
 
         # restore the logging override
         logging.disable(logging.ERROR)
-
-    @patch("audit.validators._scan_file")
-    def test_load_fixtures_federal_awards(self, mock_scan_file):
-        """load_fixtures command makes a SAC with federal awards."""
-        # make sure we have at least one user
-        User = get_user_model()
-        user, _ = User.objects.get_or_create(username="test_at_least_one")
-
-        # mock the call to the external AV service
-        mock_scan_file.return_value = MockHttpResponse(200, "clean!")
-
-        call_command("load_fixtures")
-
-        # should be a federal awards ExcelFile for this user
-        files = ExcelFile.objects.filter(
-            user=user, form_section=FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED
-        )
-        self.assertTrue(files)
-
-        # and the associated SAC has `federal_awards` data
-        self.assertTrue(files.first().sac.federal_awards)

backend/cypress/support/full-submission.js

@@ -155,7 +155,7 @@ export function testFullSubmission(isTribal, isPublic) {
     // The report ID should be found in the Completed Audits table
     cy.get('.usa-table').contains(
       'caption',
-      'The audits listed below have been submitted to the FAC for processing and may not be edited.',
+      /audits are complete/
     ).siblings().contains('td', reportId);
 
     // The Report should not be in the dissemination table

backend/dissemination/management/commands/end_to_end_workbook_test.py

@@ -1,8 +1,6 @@
-from django.apps import apps
 from django.core.management.base import BaseCommand
 from users.models import User
 import argparse
-import datetime
 import logging
 import sys
 import math
@@ -11,6 +9,7 @@
 import jwt
 import requests
 from pprint import pprint
+from datetime import datetime
 
 from dissemination.workbooklib.workbook_creation import (
     sections,
@@ -44,21 +43,13 @@
 pw.setLevel(logging.INFO)
 
 
-def step_through_certifications(sac, SAC):
-    sac.transition_name.append(SAC.STATUS.SUBMITTED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.READY_FOR_CERTIFICATION)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.AUDITOR_CERTIFIED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.AUDITEE_CERTIFIED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_name.append(SAC.STATUS.CERTIFIED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_date.append(SAC.STATUS.SUBMITTED)
-    sac.transition_date.append(datetime.datetime.today())
-    sac.transition_date.append(SAC.STATUS.DISSEMINATED)
-    sac.transition_date.append(datetime.datetime.today())
+def step_through_certifications(sac):
+    sac.transition_to_ready_for_certification()
+    sac.transition_to_auditor_certified()
+    sac.transition_to_auditee_certified()
+    sac.transition_to_submitted()
+    sac.transition_to_disseminated()
+    sac.save()
 
 
 def disseminate(sac, year):
@@ -87,7 +78,7 @@ def create_payload(api_url, role="api_fac_gov"):
     payload = {
         # PostgREST only cares about the role.
         "role": role,
-        "created": datetime.datetime.today().isoformat(),
+        "created": datetime.today().isoformat(),
     }
     return payload
 
@@ -173,6 +164,7 @@ def api_check(json_test_tables):
             equality_results = []
             for field_ndx, f in enumerate(row["fields"]):
                 # logger.info(f"Checking /{endpoint} {report_id} {f}")
+                # logger.info(f"{get_api_values(endpoint, report_id, f)}")
                 api_values = get_api_values(endpoint, report_id, f)
                 this_api_value = api_values[row_ndx]
                 this_field_value = row["values"][field_ndx]
@@ -195,7 +187,7 @@ def api_check(json_test_tables):
 
 def generate_workbooks(user, email, dbkey, year):
     entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format(
-        dbkey=dbkey, year=year, date=datetime.datetime.now()
+        dbkey=dbkey, year=year, date=datetime.now()
     )
     sac = setup_sac(user, entity_id, dbkey)
     if sac.general_information["audit_type"] == "alternative-compliance-engagement":
@@ -208,8 +200,7 @@ def generate_workbooks(user, email, dbkey, year):
             (_, json, _) = loader(fun, section)
             json_test_tables.append(json)
         _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf")
-        SingleAuditChecklist = apps.get_model("audit.SingleAuditChecklist")
-        step_through_certifications(sac, SingleAuditChecklist)
+        step_through_certifications(sac)
 
         # shaped_sac = sac_validation_shape(sac)
         # result = submission_progress_check(shaped_sac, sar=None, crossval=False)

backend/dissemination/workbooklib/federal_awards.py

@@ -126,11 +126,11 @@ def _fix_pfixes(cfdas):
     extensions = map(lambda v: ((v.cfda).split(".")[1])[:3].upper(), cfdas)
     extensions = map(
         lambda v: v
-        if re.search("^(RD|[0-9]{3}[A-Za-z]{0,1}|U[0-9]{2})$", v)
+        if re.search("^(RD|RD[0-9]|[0-9]{3}[A-Za-z]{0,1}|U[0-9]{2})$", v)
         else "000",
         extensions,
     )
-    return (prefixes, extensions)
+    return (prefixes, extensions, map(lambda v: v.cfda, cfdas))
 
 
 def _fix_passthroughs(Cfda, Passthrough, cfdas, dbkey):
@@ -213,10 +213,13 @@ def generate_federal_awards(dbkey, year, outfile):
     addls = _fix_addl_award_identification(Cfda, cfdas, dbkey)
     set_range(wb, "additional_award_identification", addls)
 
-    (prefixes, extensions) = _fix_pfixes(cfdas)
+    (prefixes, extensions, full_cfdas) = _fix_pfixes(cfdas)
     set_range(wb, "federal_agency_prefix", prefixes)
     set_range(wb, "three_digit_extension", extensions)
 
+    # We need a `cfda_key` as a magic column for the summation logic to work/be checked.
+    set_range(wb, "cfda_key", full_cfdas, conversion_fun=str)
+
     (passthrough_names, passthrough_ids) = _fix_passthroughs(
         Cfda, Passthrough, cfdas, dbkey
     )

backend/dissemination/workbooklib/sac_creation.py

@@ -259,6 +259,25 @@ def _create_test_sac(user, auditee_name, dbkey):
         email="[email protected]",  # user.email,
         role="certifying_auditor_contact",
     )
+
+    sac.auditee_certification = {}
+    sac.auditee_certification["auditee_signature"] = {}
+    sac.auditee_certification["auditee_signature"][
+        "auditee_name"
+    ] = "Bob the Auditee Name"
+    sac.auditee_certification["auditee_signature"][
+        "auditee_title"
+    ] = "Bob the Auditee Signature"
+
+    sac.auditor_certification = {}
+    sac.auditor_certification["auditor_signature"] = {}
+    sac.auditor_certification["auditor_signature"][
+        "auditor_name"
+    ] = "Alice the Auditor Name"
+    sac.auditor_certification["auditor_signature"][
+        "auditor_title"
+    ] = "Alice the Auditor Signature"
+
     sac.data_source = "TSTDAT"
     sac.save()