From dda11bdfd31a000601e427379a3fac6ee9e7f1f8 Mon Sep 17 00:00:00 2001 From: "Hassan D. M. Sambo" Date: Thu, 20 Jun 2024 16:20:24 -0400 Subject: [PATCH] 3980 address ace reports migration (#3986) * #3990 Updated schema to allow GSA migration keyword * #3980 Updated logic to prevent use of GSA migration keyword in normal intake * #3980 Added global flag for ACE report * #3980 Logic to handle ACE report and prevent error * #3980 Added test case * Linting * Bug fix * Bug fix --- backend/audit/validators.py | 7 ++ .../end_to_end_core.py | 3 + .../invalid_migration_tags.py | 2 + .../report_type_flag.py | 23 ++++++ .../sac_general_lib/audit_information.py | 68 ++++++++++++++--- .../sac_general_lib/general_information.py | 8 +- .../sac_general_lib/utils.py | 4 +- .../test_audit_information_xforms.py | 75 +++++++++++++++++++ .../sections/AuditInformation.schema.json | 73 +++++++++++++++--- .../sections/GeneralInformation.schema.json | 22 ++++-- .../GeneralInformationRequired.schema.json | 22 ++++-- backend/schemas/source/base/Base.libsonnet | 12 +++ .../sections/AuditInformation.schema.jsonnet | 49 ++++++++++-- .../GeneralInformation.schema.jsonnet | 9 ++- 14 files changed, 328 insertions(+), 49 deletions(-) create mode 100644 backend/census_historical_migration/report_type_flag.py diff --git a/backend/audit/validators.py b/backend/audit/validators.py index 24c5f97ad6..bf9e995fa4 100644 --- a/backend/audit/validators.py +++ b/backend/audit/validators.py @@ -268,6 +268,13 @@ def validate_use_of_gsa_migration_keyword_in_audit_info( if not is_data_migration and settings.GSA_MIGRATION in [ audit_information.get("is_sp_framework_required", ""), audit_information.get("is_low_risk_auditee", ""), + audit_information.get("is_going_concern_included", ""), + audit_information.get("is_internal_control_deficiency_disclosed", ""), + audit_information.get("is_internal_control_material_weakness_disclosed", ""), + audit_information.get("is_material_noncompliance_disclosed", ""), + audit_information.get("is_aicpa_audit_guide_included", ""), + ",".join(audit_information.get("agencies", [])), + ",".join(audit_information.get("gaap_results", [])), ]: raise ValidationError( _(f"{settings.GSA_MIGRATION} not permitted outside of migrations"), diff --git a/backend/census_historical_migration/end_to_end_core.py b/backend/census_historical_migration/end_to_end_core.py index f28541b4a8..800b9e72c8 100644 --- a/backend/census_historical_migration/end_to_end_core.py +++ b/backend/census_historical_migration/end_to_end_core.py @@ -35,6 +35,7 @@ from census_historical_migration.migration_result import MigrationResult from .change_record import InspectionRecord from .invalid_record import InvalidRecord +from .report_type_flag import AceFlag from django.core.exceptions import ValidationError from django.utils import timezone as django_timezone @@ -114,6 +115,7 @@ def run_end_to_end(user, audit_header): """Attempts to migrate the given audit""" InspectionRecord.reset() InvalidRecord.reset() + AceFlag.reset() try: sac = setup_sac(user, audit_header) builder_loader = workbook_builder_loader(user, sac, audit_header) @@ -236,6 +238,7 @@ def track_invalid_audit_records(audit_year, dbkey, report_id): # Save the invalid audit record and reset InvalidRecord invalid_audit_record.save() InvalidRecord.reset() + AceFlag.reset() def record_migration_status(audit_year, dbkey): diff --git a/backend/census_historical_migration/invalid_migration_tags.py b/backend/census_historical_migration/invalid_migration_tags.py index 490e12d36a..c00e557e80 100644 --- a/backend/census_historical_migration/invalid_migration_tags.py +++ b/backend/census_historical_migration/invalid_migration_tags.py @@ -11,3 +11,5 @@ class INVALID_MIGRATION_TAGS: "extra_finding_reference_numbers_in_findingstext" ) INCORRECT_FINDINGS_COUNT = "incorrect_findings_count" + + ACE_AUDIT_REPORT = "ace_audit_report" diff --git a/backend/census_historical_migration/report_type_flag.py b/backend/census_historical_migration/report_type_flag.py new file mode 100644 index 0000000000..3b7b87b129 --- /dev/null +++ b/backend/census_historical_migration/report_type_flag.py @@ -0,0 +1,23 @@ +import copy +from typing import Any + + +class AceFlag: + """Hold ACE report flag for the ongoing report migration""" + + DEFAULT: dict[str, Any] = { + "is_ace_report": False, + } + fields = copy.deepcopy(DEFAULT) + + @staticmethod + def reset(): + AceFlag.fields = copy.deepcopy(AceFlag.DEFAULT) + + @staticmethod + def set_ace_report_flag(flag): + AceFlag.fields["is_ace_report"] = flag + + @staticmethod + def get_ace_report_flag(): + return AceFlag.fields["is_ace_report"] diff --git a/backend/census_historical_migration/sac_general_lib/audit_information.py b/backend/census_historical_migration/sac_general_lib/audit_information.py index 3e0d2a0d1e..44c1f50180 100644 --- a/backend/census_historical_migration/sac_general_lib/audit_information.py +++ b/backend/census_historical_migration/sac_general_lib/audit_information.py @@ -1,11 +1,15 @@ import re from django.conf import settings + +from census_historical_migration.invalid_migration_tags import INVALID_MIGRATION_TAGS +from census_historical_migration.invalid_record import InvalidRecord +from census_historical_migration.report_type_flag import AceFlag from ..transforms.xform_string_to_int import string_to_int from ..transforms.xform_string_to_bool import string_to_bool from ..transforms.xform_string_to_string import string_to_string from ..exception_utils import DataMigrationError -from ..workbooklib.excel_creation_utils import get_audits +from ..workbooklib.excel_creation_utils import get_audits, track_invalid_records from ..base_field_maps import FormFieldMap, FormFieldInDissem from ..sac_general_lib.utils import ( create_json_from_db_object, @@ -328,17 +332,63 @@ def xform_lowrisk(audit_header): def audit_information(audit_header): """Generates audit information JSON.""" - xform_sp_framework_required(audit_header) - xform_lowrisk(audit_header) - results = xform_build_sp_framework_gaap_results(audit_header) + if AceFlag.get_ace_report_flag(): + audit_info = ace_audit_information(audit_header) + else: + xform_sp_framework_required(audit_header) + xform_lowrisk(audit_header) + results = xform_build_sp_framework_gaap_results(audit_header) + audit_info = create_json_from_db_object(audit_header, mappings) + audit_info = { + key: results.get(key, audit_info.get(key)) + for key in set(audit_info) | set(results) + } + agencies_prefixes = _get_agency_prefixes(audit_header.DBKEY, audit_header.AUDITYEAR) - audit_info = create_json_from_db_object(audit_header, mappings) - audit_info = { - key: results.get(key, audit_info.get(key)) - for key in set(audit_info) | set(results) - } audit_info["agencies"] = list(agencies_prefixes) audit.validators.validate_audit_information_json(audit_info) return audit_info + + +def ace_audit_information(audit_header): + """Constructs the audit information JSON object for an ACE report.""" + actual = create_json_from_db_object(audit_header, mappings) + default = { + "dollar_threshold": settings.GSA_MIGRATION_INT, + "gaap_results": [settings.GSA_MIGRATION], + "is_going_concern_included": settings.GSA_MIGRATION, + "is_internal_control_deficiency_disclosed": settings.GSA_MIGRATION, + "is_internal_control_material_weakness_disclosed": settings.GSA_MIGRATION, + "is_material_noncompliance_disclosed": settings.GSA_MIGRATION, + "is_aicpa_audit_guide_included": settings.GSA_MIGRATION, + "is_low_risk_auditee": settings.GSA_MIGRATION, + "agencies": [settings.GSA_MIGRATION], + } + if actual: + for key, value in actual.items(): + if value: + default[key] = value + # Tracking invalid records + invalid_records = [] + for mapping in mappings: + in_dissem = ( + mapping.in_dissem + if mapping.in_dissem != FormFieldInDissem + else mapping.in_form + ) + track_invalid_records( + [ + (mapping.in_db, getattr(audit_header, mapping.in_db)), + ], + in_dissem, + default[mapping.in_form], + invalid_records, + ) + if invalid_records: + InvalidRecord.append_invalid_migration_tag( + INVALID_MIGRATION_TAGS.ACE_AUDIT_REPORT, + ) + + return default diff --git a/backend/census_historical_migration/sac_general_lib/general_information.py b/backend/census_historical_migration/sac_general_lib/general_information.py index 3addfaaf8a..ca15b88534 100644 --- a/backend/census_historical_migration/sac_general_lib/general_information.py +++ b/backend/census_historical_migration/sac_general_lib/general_information.py @@ -21,7 +21,7 @@ from jsonschema import validate from jsonschema.exceptions import ValidationError from ..change_record import InspectionRecord, CensusRecord, GsaFacRecord - +from ..report_type_flag import AceFlag PERIOD_DICT = {"A": "annual", "B": "biennial", "O": "other"} AUDIT_TYPE_DICT = { @@ -351,10 +351,8 @@ def xform_audit_type(general_information): value_in_db = general_information["audit_type"] audit_type = _census_audit_type(value_in_db.upper()) if audit_type == AUDIT_TYPE_DICT["A"]: - raise DataMigrationError( - "Skipping ACE audit", - "skip_ace_audit", - ) + audit_type = settings.GSA_MIGRATION + AceFlag.set_ace_report_flag(True) general_information["audit_type"] = audit_type track_transformations( "AUDITTYPE", diff --git a/backend/census_historical_migration/sac_general_lib/utils.py b/backend/census_historical_migration/sac_general_lib/utils.py index 826d2599f9..a1a05ba1fb 100644 --- a/backend/census_historical_migration/sac_general_lib/utils.py +++ b/backend/census_historical_migration/sac_general_lib/utils.py @@ -2,6 +2,8 @@ from datetime import datetime, timezone import sys +from census_historical_migration.report_type_flag import AceFlag + from ..transforms.xform_string_to_string import ( string_to_string, ) @@ -22,7 +24,7 @@ def create_json_from_db_object(gobj, mappings): value = mapping.default # Fields with a value of None are skipped to maintain consistency with the logic # used in workbook data extraction and to prevent converting None into the string "None". - if value is None: + if value is None or (value == "" and AceFlag.get_ace_report_flag()): continue # Check for the type and apply the correct conversion method if mapping.type is str: diff --git a/backend/census_historical_migration/test_audit_information_xforms.py b/backend/census_historical_migration/test_audit_information_xforms.py index 5876f94e69..a5f65c4691 100644 --- a/backend/census_historical_migration/test_audit_information_xforms.py +++ b/backend/census_historical_migration/test_audit_information_xforms.py @@ -1,6 +1,8 @@ +from unittest.mock import MagicMock, patch from django.test import SimpleTestCase from .sac_general_lib.audit_information import ( + ace_audit_information, xform_build_sp_framework_gaap_results, xform_framework_basis, xform_sp_framework_required, @@ -205,3 +207,76 @@ def test_lowrisk_blank(self): audit_header.LOWRISK = "" xform_lowrisk(audit_header) self.assertEqual(audit_header.LOWRISK, settings.GSA_MIGRATION) + + +class TestAceAuditInformation(SimpleTestCase): + + def setUp(self): + self.audit_header = MagicMock() + self.audit_header.some_field = "test_value" + + self.default_values = { + "dollar_threshold": settings.GSA_MIGRATION_INT, + "gaap_results": [settings.GSA_MIGRATION], + "is_going_concern_included": settings.GSA_MIGRATION, + "is_internal_control_deficiency_disclosed": settings.GSA_MIGRATION, + "is_internal_control_material_weakness_disclosed": settings.GSA_MIGRATION, + "is_material_noncompliance_disclosed": settings.GSA_MIGRATION, + "is_aicpa_audit_guide_included": settings.GSA_MIGRATION, + "is_low_risk_auditee": settings.GSA_MIGRATION, + "agencies": [settings.GSA_MIGRATION], + } + + @patch( + "census_historical_migration.sac_general_lib.audit_information.create_json_from_db_object" + ) + def test_ace_audit_information_with_actual_values(self, mock_create_json): + """Test that the function returns the correct values when all fields are present.""" + mock_create_json.return_value = { + "dollar_threshold": 1000, + "is_low_risk_auditee": True, + "new_field": "new_value", + } + + result = ace_audit_information(self.audit_header) + + expected_result = self.default_values.copy() + expected_result.update( + { + "dollar_threshold": 1000, + "is_low_risk_auditee": True, + "new_field": "new_value", + } + ) + + self.assertEqual(result, expected_result) + + @patch( + "census_historical_migration.sac_general_lib.audit_information.create_json_from_db_object" + ) + def test_ace_audit_information_with_default_values(self, mock_create_json): + """Test that the function returns the correct values when all fields are missing.""" + mock_create_json.return_value = {} + + result = ace_audit_information(self.audit_header) + + expected_result = self.default_values + + self.assertEqual(result, expected_result) + + @patch( + "census_historical_migration.sac_general_lib.audit_information.create_json_from_db_object" + ) + def test_ace_audit_information_with_mixed_values(self, mock_create_json): + """Test that the function returns the correct values when some fields are missing.""" + mock_create_json.return_value = { + "dollar_threshold": None, + "is_low_risk_auditee": True, + } + + result = ace_audit_information(self.audit_header) + + expected_result = self.default_values.copy() + expected_result.update({"is_low_risk_auditee": True}) + + self.assertEqual(result, expected_result) diff --git a/backend/schemas/output/sections/AuditInformation.schema.json b/backend/schemas/output/sections/AuditInformation.schema.json index 6b7056ca66..c013394c25 100644 --- a/backend/schemas/output/sections/AuditInformation.schema.json +++ b/backend/schemas/output/sections/AuditInformation.schema.json @@ -83,16 +83,24 @@ "properties": { "agencies": { "items": { - "allOf": [ + "oneOf": [ { - "maxLength": 2, - "minLength": 2 + "allOf": [ + { + "maxLength": 2, + "minLength": 2 + }, + { + "pattern": "^([0-9]{2})$" + } + ], + "type": "string" }, { - "pattern": "^([0-9]{2})$" + "const": "GSA_MIGRATION", + "type": "string" } - ], - "type": "string" + ] }, "type": "array" }, @@ -107,23 +115,56 @@ "qualified_opinion", "adverse_opinion", "disclaimer_of_opinion", - "not_gaap" + "not_gaap", + "GSA_MIGRATION" ], "type": "string" }, "type": "array" }, "is_aicpa_audit_guide_included": { - "type": "boolean" + "oneOf": [ + { + "type": "boolean" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "is_going_concern_included": { - "type": "boolean" + "oneOf": [ + { + "type": "boolean" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "is_internal_control_deficiency_disclosed": { - "type": "boolean" + "oneOf": [ + { + "type": "boolean" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "is_internal_control_material_weakness_disclosed": { - "type": "boolean" + "oneOf": [ + { + "type": "boolean" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "is_low_risk_auditee": { "oneOf": [ @@ -137,7 +178,15 @@ ] }, "is_material_noncompliance_disclosed": { - "type": "boolean" + "oneOf": [ + { + "type": "boolean" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "is_sp_framework_required": { "oneOf": [ diff --git a/backend/schemas/output/sections/GeneralInformation.schema.json b/backend/schemas/output/sections/GeneralInformation.schema.json index e259f4a5ff..24edbb5a51 100644 --- a/backend/schemas/output/sections/GeneralInformation.schema.json +++ b/backend/schemas/output/sections/GeneralInformation.schema.json @@ -19,13 +19,21 @@ "type": "string" }, "audit_type": { - "description": "Type of audit being submitted", - "enum": [ - "program-specific", - "single-audit" - ], - "title": "AuditType", - "type": "string" + "oneOf": [ + { + "description": "Type of audit being submitted", + "enum": [ + "program-specific", + "single-audit" + ], + "title": "AuditType", + "type": "string" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "auditee_address_line_1": { "maxLength": 100, diff --git a/backend/schemas/output/sections/GeneralInformationRequired.schema.json b/backend/schemas/output/sections/GeneralInformationRequired.schema.json index 78a8009833..b7eeb8944b 100644 --- a/backend/schemas/output/sections/GeneralInformationRequired.schema.json +++ b/backend/schemas/output/sections/GeneralInformationRequired.schema.json @@ -129,13 +129,21 @@ "type": "string" }, "audit_type": { - "description": "Type of audit being submitted", - "enum": [ - "program-specific", - "single-audit" - ], - "title": "AuditType", - "type": "string" + "oneOf": [ + { + "description": "Type of audit being submitted", + "enum": [ + "program-specific", + "single-audit" + ], + "title": "AuditType", + "type": "string" + }, + { + "const": "GSA_MIGRATION", + "type": "string" + } + ] }, "auditee_address_line_1": { "maxLength": 100, diff --git a/backend/schemas/source/base/Base.libsonnet b/backend/schemas/source/base/Base.libsonnet index 3192b90630..5a35f2ddb6 100644 --- a/backend/schemas/source/base/Base.libsonnet +++ b/backend/schemas/source/base/Base.libsonnet @@ -222,6 +222,10 @@ local Enum = { description: 'GAAP Results (Audit Information)', enum: std.map(function(pair) pair.key, GAAP.gaap_results), }, + GAAPResults_GSAMigration: Types.string { + description: 'GAAP Results (Audit Information)', + enum: std.map(function(pair) pair.key, GAAP.gaap_results) + [Const.GSA_MIGRATION], + }, SP_Framework_Basis: Types.string { description: 'SP Framework Basis (Audit Information)', enum: std.map(function(pair) pair.key, GAAP.sp_framework_basis), @@ -381,6 +385,14 @@ local SchemaBase = Types.object { enum: ClusterNames.cluster_names + [Const.STATE_CLUSTER, Const.OTHER_CLUSTER], }, ALNPrefixes: type_aln_prefix, + ALNPrefixesWithGsaMigration: { + oneOf: [ + type_aln_prefix, + Types.string { + const: Const.GSA_MIGRATION, + }, + ], + }, ThreeDigitExtension: { oneOf: [ type_three_digit_extension, diff --git a/backend/schemas/source/sections/AuditInformation.schema.jsonnet b/backend/schemas/source/sections/AuditInformation.schema.jsonnet index ea708d381c..a3581e43fc 100644 --- a/backend/schemas/source/sections/AuditInformation.schema.jsonnet +++ b/backend/schemas/source/sections/AuditInformation.schema.jsonnet @@ -6,7 +6,7 @@ local AuditInformation = Types.object { additionalProperties: false, properties: { gaap_results: Types.array { - items: Base.Enum.GAAPResults, + items: Base.Enum.GAAPResults_GSAMigration, }, sp_framework_basis: Types.array { items: Base.Enum.SP_Framework_Basis, @@ -23,11 +23,46 @@ local AuditInformation = Types.object { }, ], }, - is_going_concern_included: Types.boolean, - is_internal_control_deficiency_disclosed: Types.boolean, - is_internal_control_material_weakness_disclosed: Types.boolean, - is_material_noncompliance_disclosed: Types.boolean, - is_aicpa_audit_guide_included: Types.boolean, + is_going_concern_included: { + oneOf: [ + Types.boolean, + Types.string { + const: Base.Const.GSA_MIGRATION, + }, + ], + }, + is_internal_control_deficiency_disclosed: { + oneOf: [ + Types.boolean, + Types.string { + const: Base.Const.GSA_MIGRATION, + }, + ], + }, + is_internal_control_material_weakness_disclosed: { + oneOf: [ + Types.boolean, + Types.string { + const: Base.Const.GSA_MIGRATION, + }, + ], + }, + is_material_noncompliance_disclosed: { + oneOf: [ + Types.boolean, + Types.string { + const: Base.Const.GSA_MIGRATION, + }, + ], + }, + is_aicpa_audit_guide_included: { + oneOf: [ + Types.boolean, + Types.string { + const: Base.Const.GSA_MIGRATION, + }, + ], + }, is_low_risk_auditee: { oneOf: [ Types.boolean, @@ -37,7 +72,7 @@ local AuditInformation = Types.object { ], }, agencies: Types.array { - items: Base.Compound.ALNPrefixes, + items: Base.Compound.ALNPrefixesWithGsaMigration, }, }, required: [ diff --git a/backend/schemas/source/sections/GeneralInformation.schema.jsonnet b/backend/schemas/source/sections/GeneralInformation.schema.jsonnet index da1a153ec1..6a1d25d56d 100644 --- a/backend/schemas/source/sections/GeneralInformation.schema.jsonnet +++ b/backend/schemas/source/sections/GeneralInformation.schema.jsonnet @@ -18,7 +18,14 @@ Typechecks fields, but allows for empty data as well. Contains conditional check auditee_fiscal_period_end: { format: 'date', }, - audit_type: Base.Enum.AuditType, + audit_type: { + oneOf: [ + Base.Enum.AuditType, + Types.string { + const: Base.Const.GSA_MIGRATION, + }, + ], + }, audit_period_covered: Base.Enum.AuditPeriod, audit_period_other_months: Base.Compound.MonthsOther,