Handle Missing entity_type and missing notes for historic audits and …

…track census data (#3746)

* Included 2019 through 2022 in the list of years when rate usage description is not required in Notes

* Handle blank entity type

* Handle missing note content

* Handle missing note title and note content

* Handle missing note title and note content

* Handle blank entity type

* Fixed linting

* Fixed linting

* Tracking blank entity_type transformation

* When Note title and content are blank, set contains_char_or_table to blank

* Update backend/census_historical_migration/sac_general_lib/general_information.py

Co-authored-by: Hassan D. M. Sambo <[email protected]>

* Update backend/census_historical_migration/sac_general_lib/general_information.py

Co-authored-by: Hassan D. M. Sambo <[email protected]>

* Updated iteration on notes

* Fixed linting

* Fixed linting

* Fixed linting

* Fixed linting

* Fixed linting

* 3763 - Track census data

* Removed unused function xform_missing_notes_records

* Renamed transform_missing_note_title_and_content to xform_missing_note_title_and_content

* Added Test cases for xform_missing_note_title_and_content

* Added Test cases for xform_missing_note_title_and_content

---------

Co-authored-by: Hassan D. M. Sambo <[email protected]>

backend/census_historical_migration/sac_general_lib/general_information.py

@@ -160,6 +160,25 @@ def xform_update_multiple_ueis_flag(audit_header):
         audit_header.MULTIPLEUEIS = "Y" if ueis else "N"
 
 
+def xform_update_entity_type(audit_header):
+    """Updates ENTITY_TYPE.
+    This updates does not propagate to the database, it only updates the object.
+    """
+    if string_to_string(audit_header.ENTITY_TYPE) == "":
+        audit_header.ENTITY_TYPE = (
+            "tribal"
+            if string_to_string(audit_header.SUPPRESSION_CODE).upper() == "IT"
+            else "unknown"
+        )
+        track_transformations(
+            "ENTITY_TYPE",
+            "",
+            "entity_type",
+            audit_header.ENTITY_TYPE,
+            "xform_update_entity_type",
+        )
+
+
 def _period_covered(s):
     """Helper to transform the period covered from Census format to FAC format."""
     if s not in PERIOD_DICT:
@@ -383,6 +402,7 @@ def general_information(audit_header):
     """Generates general information JSON."""
     xform_update_multiple_eins_flag(audit_header)
     xform_update_multiple_ueis_flag(audit_header)
+    xform_update_entity_type(audit_header)
     general_information = create_json_from_db_object(audit_header, mappings)
     transformations = [
         xform_auditee_fiscal_period_start,

backend/census_historical_migration/test_notes_to_sefa_xforms.py

@@ -2,7 +2,10 @@
 from django.test import SimpleTestCase
 
 from .exception_utils import DataMigrationError
-from .workbooklib.notes_to_sefa import xform_is_minimis_rate_used
+from .workbooklib.notes_to_sefa import (
+    xform_is_minimis_rate_used,
+    xform_missing_note_title_and_content,
+)
 
 
 class TestXformIsMinimisRateUsed(SimpleTestCase):
@@ -143,3 +146,83 @@ def test_ambiguous_or_unclear_raises_exception(self):
     def test_empty_string(self):
         """Test that the function returns GSA MIGRATION keyword when the input is an empty string."""
         self.assertEqual(xform_is_minimis_rate_used(""), settings.GSA_MIGRATION)
+
+
+class TestXformMissingNoteTitleAndContent(SimpleTestCase):
+    class MockNote:
+        def __init__(
+            self,
+            TITLE,
+            CONTENT,
+        ):
+            self.TITLE = TITLE
+            self.CONTENT = CONTENT
+
+    def _mock_notes_no_title(self):
+        notes = []
+        notes.append(
+            self.MockNote(
+                TITLE="",
+                CONTENT="SUPPORTIVE HOUSING FOR THE ELDERLY (14.157) - Balances outstanding at the end of the audit period were 3356.",
+            )
+        )
+        notes.append(
+            self.MockNote(
+                TITLE="",
+                CONTENT="MORTGAGE INSURANCE FOR THE PURCHASE OR REFINANCING OF EXISTING MULTIFAMILY HOUSING PROJECTS (14.155) - Balances outstanding at the end of the audit period were 4040.",
+            )
+        )
+        return notes
+
+    def _mock_notes_no_content(self):
+        notes = []
+        notes.append(
+            self.MockNote(
+                TITLE="Loan/loan guarantee outstanding balances",
+                CONTENT="",
+            )
+        )
+        notes.append(
+            self.MockNote(
+                TITLE="Federally Funded Insured Mortgages and Capital Advances",
+                CONTENT="",
+            )
+        )
+        return notes
+
+    def _mock_notes_with_title_content(self):
+        notes = []
+        notes.append(
+            self.MockNote(
+                TITLE="Loan/loan guarantee outstanding balances",
+                CONTENT="SUPPORTIVE HOUSING FOR THE ELDERLY (14.157) - Balances outstanding at the end of the audit period were 4000.",
+            )
+        )
+        notes.append(
+            self.MockNote(
+                TITLE="Federally Funded Insured Mortgages and Capital Advances",
+                CONTENT="MORTGAGE INSURANCE FOR THE PURCHASE OR REFINANCING OF EXISTING MULTIFAMILY HOUSING PROJECTS (14.155) - Balances outstanding at the end of the audit period were 5000.",
+            )
+        )
+        return notes
+
+    def test_note_w_no_title(self):
+        notes = self._mock_notes_no_title()
+        result = xform_missing_note_title_and_content(notes)
+        for note in result:
+            self.assertIn(settings.GSA_MIGRATION, note.TITLE)
+            self.assertNotIn(settings.GSA_MIGRATION, note.CONTENT)
+
+    def test_note_w_no_content(self):
+        notes = self._mock_notes_no_content()
+        result = xform_missing_note_title_and_content(notes)
+        for note in result:
+            self.assertNotIn(settings.GSA_MIGRATION, note.TITLE)
+            self.assertIn(settings.GSA_MIGRATION, note.CONTENT)
+
+    def test_note_with_title_content(self):
+        notes = self._mock_notes_with_title_content()
+        result = xform_missing_note_title_and_content(notes)
+        for note in result:
+            self.assertNotIn(settings.GSA_MIGRATION, note.TITLE)
+            self.assertNotIn(settings.GSA_MIGRATION, note.CONTENT)

backend/census_historical_migration/workbooklib/notes_to_sefa.py

@@ -167,28 +167,63 @@ def _get_notes(dbkey, year):
     return sort_by_field(results, "SEQ_NUMBER")
 
 
-def xform_missing_notes_records(audit_header, policies_content, rate_content):
-    """Transforms missing notes records for 2016, 2017, and 2018 audits."""
-    if string_to_string(audit_header.AUDITYEAR) in ["2018", "2017", "2016"] and not (
-        policies_content or rate_content
-    ):
-        policies_content = settings.GSA_MIGRATION
-        rate_content = settings.GSA_MIGRATION
+def xform_missing_notes_records_v2(audit_header, policies_content, rate_content):
+    """Transforms missing notes records for 2022, 2021, 2020, 2019, 2016, 2017 and 2018 audits.
+    Note:
+    This function replaces xform_missing_notes_records function.
+    This function covers all years 2016 through 2022.
+    This function tracks census data for policies_content and rate_content."""
+
+    if string_to_string(audit_header.AUDITYEAR) in [
+        "2022",
+        "2021",
+        "2020",
+        "2019",
+        "2018",
+        "2017",
+        "2016",
+    ] and not (policies_content or rate_content):
         track_data_transformation(
             policies_content,
             settings.GSA_MIGRATION,
-            "xform_missing_notes_records",
+            "xform_missing_notes_records_v2",
             "accounting_policies",
         )
+        policies_content = settings.GSA_MIGRATION
         track_data_transformation(
             rate_content,
             settings.GSA_MIGRATION,
-            "xform_missing_notes_records",
+            "xform_missing_notes_records_v2",
             "rate_explained",
         )
+        rate_content = settings.GSA_MIGRATION
     return policies_content, rate_content
 
 
+def xform_missing_note_title_and_content(notes):
+    """Transforms missing note title and note content."""
+    for note in notes:
+        if string_to_string(note.TITLE) == "" and string_to_string(note.CONTENT) != "":
+            track_data_transformation(
+                note.TITLE,
+                settings.GSA_MIGRATION,
+                "xform_missing_note_title_and_content",
+                "note_title",
+            )
+            note.TITLE = settings.GSA_MIGRATION
+
+        if string_to_string(note.CONTENT) == "" and string_to_string(note.TITLE) != "":
+            track_data_transformation(
+                note.CONTENT,
+                settings.GSA_MIGRATION,
+                "xform_missing_note_title_and_content",
+                "content",
+            )
+            note.CONTENT = settings.GSA_MIGRATION
+
+    return notes
+
+
 def generate_notes_to_sefa(audit_header, outfile):
     """
     Generates notes to SEFA workbook for a given audit header.
@@ -201,21 +236,27 @@ def generate_notes_to_sefa(audit_header, outfile):
     uei = xform_retrieve_uei(audit_header.UEI)
     set_workbook_uei(wb, uei)
     notes = _get_notes(audit_header.DBKEY, audit_header.AUDITYEAR)
+    notes = xform_missing_note_title_and_content(notes)
     rate_content, index = _get_minimis_cost_rate(
         audit_header.DBKEY, audit_header.AUDITYEAR
     )
     policies_content = _get_accounting_policies(
         audit_header.DBKEY, audit_header.AUDITYEAR
     )
     is_minimis_rate_used = xform_is_minimis_rate_used(rate_content, index)
-    policies_content, rate_content = xform_missing_notes_records(
+    policies_content, rate_content = xform_missing_notes_records_v2(
         audit_header, policies_content, rate_content
     )
     set_range(wb, "accounting_policies", [policies_content])
     set_range(wb, "is_minimis_rate_used", [is_minimis_rate_used])
     set_range(wb, "rate_explained", [rate_content])
 
-    contains_chart_or_tables = [settings.GSA_MIGRATION] * len(notes)
+    contains_chart_or_tables = []
+    for note in notes:
+        if string_to_string(note.TITLE) == "" and string_to_string(note.CONTENT) == "":
+            contains_chart_or_tables.append("")
+        else:
+            contains_chart_or_tables.append(settings.GSA_MIGRATION)
 
     # Map the rest as notes.
     map_simple_columns(wb, mappings, notes)