From dc47bb507b31c8f0dc50997a038ed7deabc68f8c Mon Sep 17 00:00:00 2001 From: Sudha Kumar <135276194+gsa-suk@users.noreply.github.com> Date: Mon, 29 Apr 2024 08:45:02 -0700 Subject: [PATCH] Handle Missing entity_type and missing notes for historic audits and track census data (#3746) * Included 2019 through 2022 in the list of years when rate usage description is not required in Notes * Handle blank entity type * Handle missing note content * Handle missing note title and note content * Handle missing note title and note content * Handle blank entity type * Fixed linting * Fixed linting * Tracking blank entity_type transformation * When Note title and content are blank, set contains_char_or_table to blank * Update backend/census_historical_migration/sac_general_lib/general_information.py Co-authored-by: Hassan D. M. Sambo * Update backend/census_historical_migration/sac_general_lib/general_information.py Co-authored-by: Hassan D. M. Sambo * Updated iteration on notes * Fixed linting * Fixed linting * Fixed linting * Fixed linting * Fixed linting * 3763 - Track census data * Removed unused function xform_missing_notes_records * Renamed transform_missing_note_title_and_content to xform_missing_note_title_and_content * Added Test cases for xform_missing_note_title_and_content * Added Test cases for xform_missing_note_title_and_content --------- Co-authored-by: Hassan D. M. Sambo --- .../sac_general_lib/general_information.py | 20 +++++ .../test_notes_to_sefa_xforms.py | 85 ++++++++++++++++++- .../workbooklib/notes_to_sefa.py | 63 +++++++++++--- 3 files changed, 156 insertions(+), 12 deletions(-) diff --git a/backend/census_historical_migration/sac_general_lib/general_information.py b/backend/census_historical_migration/sac_general_lib/general_information.py index 318865bc9b..1377dc5c94 100644 --- a/backend/census_historical_migration/sac_general_lib/general_information.py +++ b/backend/census_historical_migration/sac_general_lib/general_information.py @@ -160,6 +160,25 @@ def xform_update_multiple_ueis_flag(audit_header): audit_header.MULTIPLEUEIS = "Y" if ueis else "N" +def xform_update_entity_type(audit_header): + """Updates ENTITY_TYPE. + This updates does not propagate to the database, it only updates the object. + """ + if string_to_string(audit_header.ENTITY_TYPE) == "": + audit_header.ENTITY_TYPE = ( + "tribal" + if string_to_string(audit_header.SUPPRESSION_CODE).upper() == "IT" + else "unknown" + ) + track_transformations( + "ENTITY_TYPE", + "", + "entity_type", + audit_header.ENTITY_TYPE, + "xform_update_entity_type", + ) + + def _period_covered(s): """Helper to transform the period covered from Census format to FAC format.""" if s not in PERIOD_DICT: @@ -383,6 +402,7 @@ def general_information(audit_header): """Generates general information JSON.""" xform_update_multiple_eins_flag(audit_header) xform_update_multiple_ueis_flag(audit_header) + xform_update_entity_type(audit_header) general_information = create_json_from_db_object(audit_header, mappings) transformations = [ xform_auditee_fiscal_period_start, diff --git a/backend/census_historical_migration/test_notes_to_sefa_xforms.py b/backend/census_historical_migration/test_notes_to_sefa_xforms.py index ffce032cfb..c6d18afd93 100644 --- a/backend/census_historical_migration/test_notes_to_sefa_xforms.py +++ b/backend/census_historical_migration/test_notes_to_sefa_xforms.py @@ -2,7 +2,10 @@ from django.test import SimpleTestCase from .exception_utils import DataMigrationError -from .workbooklib.notes_to_sefa import xform_is_minimis_rate_used +from .workbooklib.notes_to_sefa import ( + xform_is_minimis_rate_used, + xform_missing_note_title_and_content, +) class TestXformIsMinimisRateUsed(SimpleTestCase): @@ -143,3 +146,83 @@ def test_ambiguous_or_unclear_raises_exception(self): def test_empty_string(self): """Test that the function returns GSA MIGRATION keyword when the input is an empty string.""" self.assertEqual(xform_is_minimis_rate_used(""), settings.GSA_MIGRATION) + + +class TestXformMissingNoteTitleAndContent(SimpleTestCase): + class MockNote: + def __init__( + self, + TITLE, + CONTENT, + ): + self.TITLE = TITLE + self.CONTENT = CONTENT + + def _mock_notes_no_title(self): + notes = [] + notes.append( + self.MockNote( + TITLE="", + CONTENT="SUPPORTIVE HOUSING FOR THE ELDERLY (14.157) - Balances outstanding at the end of the audit period were 3356.", + ) + ) + notes.append( + self.MockNote( + TITLE="", + CONTENT="MORTGAGE INSURANCE FOR THE PURCHASE OR REFINANCING OF EXISTING MULTIFAMILY HOUSING PROJECTS (14.155) - Balances outstanding at the end of the audit period were 4040.", + ) + ) + return notes + + def _mock_notes_no_content(self): + notes = [] + notes.append( + self.MockNote( + TITLE="Loan/loan guarantee outstanding balances", + CONTENT="", + ) + ) + notes.append( + self.MockNote( + TITLE="Federally Funded Insured Mortgages and Capital Advances", + CONTENT="", + ) + ) + return notes + + def _mock_notes_with_title_content(self): + notes = [] + notes.append( + self.MockNote( + TITLE="Loan/loan guarantee outstanding balances", + CONTENT="SUPPORTIVE HOUSING FOR THE ELDERLY (14.157) - Balances outstanding at the end of the audit period were 4000.", + ) + ) + notes.append( + self.MockNote( + TITLE="Federally Funded Insured Mortgages and Capital Advances", + CONTENT="MORTGAGE INSURANCE FOR THE PURCHASE OR REFINANCING OF EXISTING MULTIFAMILY HOUSING PROJECTS (14.155) - Balances outstanding at the end of the audit period were 5000.", + ) + ) + return notes + + def test_note_w_no_title(self): + notes = self._mock_notes_no_title() + result = xform_missing_note_title_and_content(notes) + for note in result: + self.assertIn(settings.GSA_MIGRATION, note.TITLE) + self.assertNotIn(settings.GSA_MIGRATION, note.CONTENT) + + def test_note_w_no_content(self): + notes = self._mock_notes_no_content() + result = xform_missing_note_title_and_content(notes) + for note in result: + self.assertNotIn(settings.GSA_MIGRATION, note.TITLE) + self.assertIn(settings.GSA_MIGRATION, note.CONTENT) + + def test_note_with_title_content(self): + notes = self._mock_notes_with_title_content() + result = xform_missing_note_title_and_content(notes) + for note in result: + self.assertNotIn(settings.GSA_MIGRATION, note.TITLE) + self.assertNotIn(settings.GSA_MIGRATION, note.CONTENT) diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index a095636601..f2a066b914 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -167,28 +167,63 @@ def _get_notes(dbkey, year): return sort_by_field(results, "SEQ_NUMBER") -def xform_missing_notes_records(audit_header, policies_content, rate_content): - """Transforms missing notes records for 2016, 2017, and 2018 audits.""" - if string_to_string(audit_header.AUDITYEAR) in ["2018", "2017", "2016"] and not ( - policies_content or rate_content - ): - policies_content = settings.GSA_MIGRATION - rate_content = settings.GSA_MIGRATION +def xform_missing_notes_records_v2(audit_header, policies_content, rate_content): + """Transforms missing notes records for 2022, 2021, 2020, 2019, 2016, 2017 and 2018 audits. + Note: + This function replaces xform_missing_notes_records function. + This function covers all years 2016 through 2022. + This function tracks census data for policies_content and rate_content.""" + + if string_to_string(audit_header.AUDITYEAR) in [ + "2022", + "2021", + "2020", + "2019", + "2018", + "2017", + "2016", + ] and not (policies_content or rate_content): track_data_transformation( policies_content, settings.GSA_MIGRATION, - "xform_missing_notes_records", + "xform_missing_notes_records_v2", "accounting_policies", ) + policies_content = settings.GSA_MIGRATION track_data_transformation( rate_content, settings.GSA_MIGRATION, - "xform_missing_notes_records", + "xform_missing_notes_records_v2", "rate_explained", ) + rate_content = settings.GSA_MIGRATION return policies_content, rate_content +def xform_missing_note_title_and_content(notes): + """Transforms missing note title and note content.""" + for note in notes: + if string_to_string(note.TITLE) == "" and string_to_string(note.CONTENT) != "": + track_data_transformation( + note.TITLE, + settings.GSA_MIGRATION, + "xform_missing_note_title_and_content", + "note_title", + ) + note.TITLE = settings.GSA_MIGRATION + + if string_to_string(note.CONTENT) == "" and string_to_string(note.TITLE) != "": + track_data_transformation( + note.CONTENT, + settings.GSA_MIGRATION, + "xform_missing_note_title_and_content", + "content", + ) + note.CONTENT = settings.GSA_MIGRATION + + return notes + + def generate_notes_to_sefa(audit_header, outfile): """ Generates notes to SEFA workbook for a given audit header. @@ -201,6 +236,7 @@ def generate_notes_to_sefa(audit_header, outfile): uei = xform_retrieve_uei(audit_header.UEI) set_workbook_uei(wb, uei) notes = _get_notes(audit_header.DBKEY, audit_header.AUDITYEAR) + notes = xform_missing_note_title_and_content(notes) rate_content, index = _get_minimis_cost_rate( audit_header.DBKEY, audit_header.AUDITYEAR ) @@ -208,14 +244,19 @@ def generate_notes_to_sefa(audit_header, outfile): audit_header.DBKEY, audit_header.AUDITYEAR ) is_minimis_rate_used = xform_is_minimis_rate_used(rate_content, index) - policies_content, rate_content = xform_missing_notes_records( + policies_content, rate_content = xform_missing_notes_records_v2( audit_header, policies_content, rate_content ) set_range(wb, "accounting_policies", [policies_content]) set_range(wb, "is_minimis_rate_used", [is_minimis_rate_used]) set_range(wb, "rate_explained", [rate_content]) - contains_chart_or_tables = [settings.GSA_MIGRATION] * len(notes) + contains_chart_or_tables = [] + for note in notes: + if string_to_string(note.TITLE) == "" and string_to_string(note.CONTENT) == "": + contains_chart_or_tables.append("") + else: + contains_chart_or_tables.append(settings.GSA_MIGRATION) # Map the rest as notes. map_simple_columns(wb, mappings, notes)