diff --git a/backend/audit/test_workbooks_should_pass.py b/backend/audit/test_workbooks_should_pass.py index 9e56bbb272..4b72294d64 100644 --- a/backend/audit/test_workbooks_should_pass.py +++ b/backend/audit/test_workbooks_should_pass.py @@ -56,6 +56,21 @@ def map_file_to_extractor_validator(filename): return (None, None) +def process_workbook_set(workbook_set_path): + """Process each workbook set in the given path.""" + for wb_path, _, wb_files in os.walk(workbook_set_path): + for file in wb_files: + if re.search("xlsx$", str(file)): + full_path = os.path.join(wb_path, file) + (extractor, validator) = map_file_to_extractor_validator(full_path) + if extractor: + print(f"Extracting and validating {file}") + ir = extractor(full_path) + validator(ir) + else: + print(f"No extractor found for [{file}]") + + class PassingWorkbooks(SimpleTestCase): def test_passing_workbooks(self): workbook_sets = reduce( @@ -64,21 +79,7 @@ def test_passing_workbooks(self): for dirpath, dirnames, _ in os.walk(workbook_sets): for workbook_set in dirnames: print("Walking ", workbook_set) - for wb_path, _, wb_files in os.walk( - os.path.join(dirpath, workbook_set) - ): - for file in wb_files: - if re.search("xlsx$", str(file)): - full_path = os.path.join(wb_path, file) - (extractor, validator) = map_file_to_extractor_validator( - full_path - ) - if extractor: - print(f"Extracting and validating {file}") - ir = extractor(full_path) - validator(ir) - else: - print(f"No extractor found for [{file}]") + process_workbook_set(os.path.join(dirpath, workbook_set)) # Can't run this when subclased from SimpleTestCase. # Can't write to the database `default`. diff --git a/backend/census_historical_migration/README.md b/backend/census_historical_migration/README.md index c07f15e3ec..95570f276e 100644 --- a/backend/census_historical_migration/README.md +++ b/backend/census_historical_migration/README.md @@ -56,11 +56,10 @@ docker compose run --rm web python manage.py csv_to_postgres --folder data --chu ### How to run the historic data migrator: ``` -docker compose run --rm web python manage.py historic_data_migrator --email any_email_in_the_system@woo.gov \ +docker compose run --rm web python manage.py historic_data_migrator --years 22 \ - --dbkeys 100010 + --dbkeys 177310 ``` -- The email address currently must be a User in the system. As this has only been run locally so far, it would often be a test account in my local sandbox env. - `year` and `dbkey` are optional. The script will use default values for these if they aren't provided. ### How to run the historic workbook generator: @@ -68,7 +67,7 @@ docker compose run --rm web python manage.py historic_data_migrator --email any_ docker compose run --rm web python manage.py historic_workbook_generator \ --year 22 \ --output \ - --dbkey 100010 + --dbkey 177310 ``` - `year` is optional and defaults to `22`. - The `output` directory will be created if it doesn't already exist. diff --git a/backend/census_historical_migration/admin.py b/backend/census_historical_migration/admin.py index 8f217f70e8..e32ef50ae4 100644 --- a/backend/census_historical_migration/admin.py +++ b/backend/census_historical_migration/admin.py @@ -1,3 +1,29 @@ from django.contrib import admin # noqa: F401 -# Register your models here. +from census_historical_migration.models import ( + ELECAUDITHEADER, + ELECEINS, + ELECAUDITFINDINGS, + ELECNOTES, + ELECFINDINGSTEXT, + ELECCPAS, + ELECAUDITS, + ELECPASSTHROUGH, + ELECUEIS, + ELECCAPTEXT, + ReportMigrationStatus, + MigrationErrorDetail, +) + +admin.site.register(ELECAUDITHEADER) +admin.site.register(ELECEINS) +admin.site.register(ELECAUDITFINDINGS) +admin.site.register(ELECNOTES) +admin.site.register(ELECFINDINGSTEXT) +admin.site.register(ELECCPAS) +admin.site.register(ELECAUDITS) +admin.site.register(ELECPASSTHROUGH) +admin.site.register(ELECUEIS) +admin.site.register(ELECCAPTEXT) +admin.site.register(ReportMigrationStatus) +admin.site.register(MigrationErrorDetail) diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx new file mode 100644 index 0000000000..fb5bfeb68e Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx new file mode 100644 index 0000000000..fe3d82ab5b Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx new file mode 100644 index 0000000000..7f098f1659 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx new file mode 100644 index 0000000000..86dec4b8b9 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx new file mode 100644 index 0000000000..9ef7b6e16c Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx new file mode 100644 index 0000000000..53533bb2f8 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx new file mode 100644 index 0000000000..ad8d8fc532 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx new file mode 100644 index 0000000000..8ea4f7592c Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/test-array-177310.json b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/test-array-177310.json new file mode 100644 index 0000000000..09d5a0ca4a --- /dev/null +++ b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/test-array-177310.json @@ -0,0 +1,1583 @@ +[ + { + "endpoint": "additional_eins", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [], + "singletons": { + "auditee_uei": "LJNSJ1C1KU15" + } + }, + { + "endpoint": "additional_ueis", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [], + "singletons": { + "auditee_uei": "LJNSJ1C1KU15" + } + }, + { + "endpoint": "corrective_action_plans", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [ + { + "fields": [ + "finding_ref_number", + "planned_action", + "contains_chart_or_table" + ], + "values": [ + "2022-001", + "CORRECTIVE ACTION PLAN FOR FINDINGS REPORTED UNDER UNIFORM GUIDANCECentral Valley School District No. 356September 1, 2021 through August 31, 2022This schedule presents the corrective action planned by the District for findings reported in this report in accordance with Title 2 U.S. Code of Federal Regulations (CFR) Part 200, Uniform Administrative Requirements, Cost Principles, and Audit Requirements for Federal Awards (Uniform Guidance).Finding ref number:2022-001Finding caption:The District did not have adequate internal controls for ensuring compliance with wage rate requirements.Name, address, and telephone of District contact person:Mathew Knott, Director of Business Services2218 N. Molter RoadLiberty Lake, WA 99019509-558-5437Corrective action the auditee plans to take in response to the finding:The District agrees with the State Auditors Office that we did not have adequate internal controls for ensuring compliance with federal prevailing wage rate requirements as noted. The District used the same process as noted in this Finding in the prior audit which did not have any exceptions noted by the State Auditors Office. Moving forward the District will ensure federal prevailing wage rate clauses are in contracts entered into using federal funds and that weekly certified payroll reports are collected from contractors and subcontractors.Anticipated date to complete the corrective action: August 2023", + "N" + ] + } + ], + "singletons": { + "auditee_uei": "LJNSJ1C1KU15" + } + }, + { + "endpoint": "federal_awards", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [ + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "553", + "SCHOOL BREAKFAST PROGRAM", + "nan", + 744086, + "nan", + 7103433, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 744086, + "10", + "553", + "AWARD-0001", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "SPECIAL EDUCATION_GRANTS TO STATES", + "nan", + 3507525, + "nan", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 108754, + "84", + "027", + "AWARD-0002", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "COVID 19 - SPECIAL EDUCATION GRANTS TO STATES", + "nan", + 3507525, + "COVID 19", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 639680, + "84", + "027", + "AWARD-0003", + "COVID 19", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "SPECIAL EDUCATION_PRESCHOOL GRANTS", + "nan", + 128501, + "nan", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 49577, + "84", + "173", + "AWARD-0004", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "SPECIAL EDUCATION_PRESCHOOL GRANTS", + "nan", + 128501, + "nan", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 5001, + "84", + "173", + "AWARD-0005", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "SPECIAL EDUCATION_PRESCHOOL GRANTS", + "nan", + 128501, + "nan", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 5361, + "84", + "173", + "AWARD-0006", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "SPECIAL EDUCATION_PRESCHOOL GRANTS", + "nan", + 128501, + "nan", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 15000, + "84", + "173", + "AWARD-0007", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "COVID 19 - SPECIAL EDUCATION PRESCHOOL GRANTS", + "nan", + 128501, + "COVID 19", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 53562, + "84", + "173", + "AWARD-0008", + "COVID 19", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "048", + "CAREER AND TECHNICAL EDUCATION -- BASIC GRANTS TO STATES", + "nan", + 91269, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 91269, + "84", + "048", + "AWARD-0009", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "196", + "EDUCATION FOR HOMELESS CHILDREN AND YOUTH", + "nan", + 57915, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 57915, + "84", + "196", + "AWARD-0010", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "365", + "ENGLISH LANGUAGE ACQUISITION STATE GRANTS", + "nan", + 39801, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 39801, + "84", + "365", + "AWARD-0011", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "555", + "NATIONAL SCHOOL LUNCH PROGRAM", + "nan", + 6275850, + "nan", + 7103433, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 439819, + "10", + "555", + "AWARD-0012", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "367", + "SUPPORTING EFFECTIVE INSTRUCTION STATE GRANTS (FORMERLY IMPROVING TEACHER QUALITY STATE GRANTS)", + "nan", + 364437, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 364437, + "84", + "367", + "AWARD-0013", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "COVID 19 - EDUCATION STABILIZATION FUND", + "nan", + 22519443, + "COVID-19, 84.425U", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 1, + 31894, + "84", + "425", + "AWARD-0014", + "COVID-19, 84.425U", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "COVID 19 - EDUCATION STABILIZATION FUND", + "nan", + 22519443, + "COVID-19, 84.425D", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 1, + 3648641, + "84", + "425", + "AWARD-0015", + "COVID-19, 84.425D", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "COVID 19 - EDUCATION STABILIZATION FUND", + "nan", + 22519443, + "COVID-19, 84.425U", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 1, + 16632311, + "84", + "425", + "AWARD-0016", + "COVID-19, 84.425U", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "COVID 19 - EDUCATION STABILIZATION FUND", + "nan", + 22519443, + "COVID-19, 84.425W", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 1, + 13809, + "84", + "425", + "AWARD-0017", + "COVID-19, 84.425W", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "COVID 19 - EDUCATION STABILIZATION FUND", + "nan", + 22519443, + "COVID-19, 84.425W", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 1, + 6430, + "84", + "425", + "AWARD-0018", + "COVID-19, 84.425W", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "COVID 19 - EDUCATION STABILIZATION FUND", + "nan", + 22519443, + "COVID-19, 84.425U", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 1, + 2186358, + "84", + "425", + "AWARD-0019", + "COVID-19, 84.425U", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "555", + "NATIONAL SCHOOL LUNCH PROGRAM", + "nan", + 6275850, + "nan", + 7103433, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 5836031, + "10", + "555", + "AWARD-0020", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "559", + "SUMMER FOOD SERVICE PROGRAM FOR CHILDREN", + "nan", + 26895, + "nan", + 7103433, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 26895, + "10", + "559", + "AWARD-0021", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "582", + "FRESH FRUIT AND VEGETABLE PROGRAM", + "nan", + 56602, + "nan", + 7103433, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 56602, + "10", + "582", + "AWARD-0022", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "558", + "CHILD AND ADULT CARE FOOD PROGRAM", + "nan", + 173463, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 173463, + "10", + "558", + "AWARD-0023", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "649", + "COVID 19 - PANDEMIC EBT ADMINISTRATIVE COSTS", + "nan", + 5814, + "COVID 19", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 5814, + "10", + "649", + "AWARD-0024", + "COVID 19", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "010", + "TITLE I GRANTS TO LOCAL EDUCATIONAL AGENCIES", + "nan", + 2680846, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 0, + 2680846, + "84", + "010", + "AWARD-0025", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "SPECIAL EDUCATION_GRANTS TO STATES", + "nan", + 3507525, + "nan", + 3636026, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 2759091, + "84", + "027", + "AWARD-0026", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + } + ], + "singletons": { + "auditee_uei": "LJNSJ1C1KU15", + "total_amount_expended": 36672447 + } + }, + { + "endpoint": "findings_text", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [ + { + "fields": [ + "finding_ref_number", + "finding_text", + "contains_chart_or_table" + ], + "values": [ + "2022-001", + "SCHEDULE OF FEDERAL AWARD FINDINGS AND QUESTIONED COSTSCentral Valley School District No. 356September 1, 2021 through August 31, 20222022-001The District did not have adequate internal controls for ensuring compliance with wage rate requirements.Assistance Listing Number and Title:84.425, COVID-19 Education Stabilization FundFederal Grantor Name:U.S. Department of EducationFederal Award/Contract Number:N/APass-through Entity Name:Office of Superintendent of Public Instruction Pass-through Award/Contract Number:COVID-19, 84.845U-0712260COVID-19, 84.425D-0120411 COVID-19, 84.425U-0138118COVID-19, 84.425W-0459044COVID-19, 84.425W-0459641COVID-19, 84.425U-0137146Known Questioned Cost Amount:$0BackgroundThe objectives of the Education Stabilization Fund (ESF) program are to prevent, prepare for and respond to the COVID-19 pandemic. In fiscal year 2022, the District expended a total of $22,519,443 of its ESF awards. This included $3,648,641 in the Elementary and Secondary School Emergency Relief Fund (ESSER II) subprogram (84.425D), $18,850,563 in the American Rescue Plan Elementary and Secondary School Emergency Relief (ARP ESSER/ESSER III) subprogram (84.425U), and $20,239 in the American Rescue Plan Elementary and Secondary School Emergency Relief Homeless Children and Youth (ARP HCY I & II) subprogram (84.425W).Federal regulations require recipients to establish and maintain internal controls that ensure compliance with program requirements. These controls include understanding grant requirements and monitoring the effectiveness of established controls.Under federal wage rate requirements, also known as the Davis-Bacon Act, contractors and subcontractors that work on projects financed with more than $2,000 of federal money must pay laborers and mechanics wage rates that the U.S. Department of Labor considers being similar to what local workers have been paid for similar projects.For construction contracts subject to these wage rate requirements, the District must include a provision that the contractor and subcontractor comply with those requirements and the Department of Labors regulations. This includes a requirement for the contractor and its subcontractor to submit to the District weekly, for each week in which any contract work is performed, certified payroll reports. These reports must include a copy of the payroll and a signed statement of compliance.The District may use a contracted project manager to collect certified payroll reports from contractors and subcontractors, but ultimately, it is the Districts responsibility to comply with these requirements and maintain documentation demonstrating compliance.Description of ConditionDuring the 2021-22 school year, the District spent $2,685,727 from its ESSER II award to pay two contractors and their subcontractors for installing and upgrading the heating, ventilation and air conditioning systems at four school buildings and the District office. These projects were part of the Districts school facility capital improvement efforts to prevent the spread of COVID-19 and enable school operations by facilitating greater air flow and filtration. During the audit period, the District awarded and entered into a contract with one of the contractors, and contracted with two project managers to supervise the projects, which included collecting weekly certified payroll reports from the contractors and subcontractors.Our audit found the District did not have adequate internal controls for ensuring compliance with federal prevailing wage rate requirements. Specifically, the District did not:Include the required prevailing wage rate clauses in the contract established with one contractorCollect, or ensure project managers collected, weekly certified payroll reports from the contractors and their subcontractors to confirm they paid laborers proper prevailing wagesWe consider these deficiencies in internal controls to be a material weakness, which led to material noncompliance.The issue was not reported as a finding in the prior audit.Cause of ConditionDistrict staff said they used a purchasing cooperative to procure the general contractor, and they relied on the contract between the purchasing cooperative and the general contractor, which included the required prevailing wage language. However, staff said they did not know the District was required to include this language in its own contract with the contractor.Additionally, the District relied on the project managers to verify the contractors and subcontractors submitted weekly certified payroll reports to the Washington State Labor and Industries (L&I) website. Although District staff said they checked that weekly certified payroll reports were uploaded to the L&I system before the District paid the contractors, this process was not documented. Further, District staff and project managers did not know the District needed to obtain all certified payroll reports each week.Effect of Condition Without adequate internal controls that ensure it includes the prevailing wage rate clauses in its contracts and collects all weekly certified payroll reports, the District cannot demonstrate it complied with federal wage rate requirements. The District could also be liable for paying any additional wages if the contractors and subcontractors did not pay prevailing wage rates to laborers working on the contracts.RecommendationWe recommend the District develop internal controls to ensure compliance with federal wage rate requirements. This should include inserting prevailing wage rate clauses into contracts, as well as implementing effective monitoring processes to collect and review all weekly certified payroll reports timely from contractors and subcontractors.Districts ResponseThe District agrees with the State Auditors Office that we did not have adequate internal controls for ensuring compliance with federal prevailing wage rate requirements as noted. The District used the same process as noted in this Finding in the prior audit which did not have any exceptions noted by the State Auditors Office. Moving forward the District will ensure federal prevailing wage rate clauses are in contracts entered into using federal funds and that weekly certified payroll reports are collected from contractors and subcontractors.Auditors RemarksWe appreciate the Districts commitment to resolve this finding and thank the District for its cooperation and assistance during the audit. We will review the status of the Districts corrective action during our next audit.Applicable Laws and RegulationsTitle 2 U.S. Code of Federal Regulations (CFR) Part 200, Uniform Administrative Requirements, Cost Principles, and Audit Requirements for Federal Awards (Uniform Guidance), section 516, Audit findings, establishes reporting requirements for audit findings.Title 2 CFR Part 200, Uniform Guidance, section 303, Internal controls, describes the requirements for auditees to maintain internal controls over federal programs and comply with federal program requirements.The American Institute of Certified Public Accountants defines significant deficiencies and material weaknesses in its Codification of Statements on Auditing Standards, section 935, Compliance Audits, paragraph 11.Title 29 CFR, Section 3.3 Weekly statement with respect to payment of wages, and Section 3.4 Submission of weekly statements and the preservation and inspection of weekly payroll records, establishes requirements for contractor or subcontractor submission of weekly certified payroll reports.Title 29 CFR, Section 5.5 Contract provisions and related matters establishes the requirements for the contracting officer to insert in full any contract in excess of $2,000 which is entered into for the actual construction, alteration and/or repair, including painting and decorating, of a public building or public work, or building or work financed in whole or in part with federal funds the clauses listed, which includes but is not limited to the minimum wages to be paid and payrolls and basic records to be maintained (submission of weekly certified payrolls)", + "N" + ] + } + ], + "singletons": { + "auditee_uei": "LJNSJ1C1KU15" + } + }, + { + "endpoint": "findings", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [ + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "N", + "2022-001", + "N", + "Y", + "Y", + "N", + "N", + "N", + "N", + "AWARD-0015" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "N", + "2022-001", + "N", + "Y", + "Y", + "N", + "N", + "N", + "N", + "AWARD-0017" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "N", + "2022-001", + "N", + "Y", + "Y", + "N", + "N", + "N", + "N", + "AWARD-0019" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "N", + "2022-001", + "N", + "Y", + "Y", + "N", + "N", + "N", + "N", + "AWARD-0014" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "N", + "2022-001", + "N", + "Y", + "Y", + "N", + "N", + "N", + "N", + "AWARD-0016" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "N", + "2022-001", + "N", + "Y", + "Y", + "N", + "N", + "N", + "N", + "AWARD-0018" + ] + } + ], + "singletons": {} + }, + { + "endpoint": "notes_to_sefa", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [ + { + "fields": [ + "title", + "content" + ], + "values": [ + "NOTE 3 - PROGRAM COST/MATCHING CONTRIBUTIONS", + "The amounts shown as current year expenses represent only the federal grant portion of the program costs. Entire program costs, including the Central Valley School District's local matching share, may be more than shown. Such expenditures are recognized following, the cost principles contained in the Title 2 U.S. Code of Federal Regulations Part 200, Uniform Administrative Requirements, Cost Principles, and Audit Requirements for Federal Awards, wherein certain types of expenditures are not allowable or are limited as to reimbursement." + ] + }, + { + "fields": [ + "title", + "content" + ], + "values": [ + "NOTE 4 - NONCASH AWARDS", + "The amount of commodities reported on the schedule is the value of commodities distributed by the Central Valley School District during the current year and priced as prescribed by OSPI Child Nutrition Services & USDA." + ] + }, + { + "fields": [ + "title", + "content" + ], + "values": [ + "NOTE 5 - TRANSFERABILITY", + "As allowed by federal regulations, the Central Valley School District elected to transfer program funds. The district expended $216,008 from its Title IV, Part A Student Support and Academic Enrichment (84.424) on allowable activities of the Title II, Part A Supporting Effective Instruction State Grants (84.367). This amount is reflected in the expenditures of Title II, Part A Supporting Effective Instruction State Grants (84.367)." + ] + } + ], + "singletons": { + "accounting_policies": "NOTE 1 - BASIS OF ACCOUNTING: This Schedule is prepared on the same basis of accounting as the Central Valley School District's financial statements. The Central Valley School District uses the modified accrual basis of accounting. Expenditures represent only the federally funded portions of the program. District records should be consulted to determine amounts expended or matched from non-federal sources.", + "auditee_uei": "LJNSJ1C1KU15", + "is_minimis_rate_used": "N", + "rate_explained": "NOTE 2 - FEDERAL DE MINUS INDIRECT RATE: The Central Valley School District has not elected to use the 10-percent de minimis indirect cost rate allowed under the Uniform Guidance. The Central Valley School District used the federal restricted rate of 2.72% and federal unrestricted rate of 10.82%." + } + }, + { + "endpoint": "secondary_auditor", + "report_id": "2022-08-CENSUS-0000177310", + "rows": [], + "singletons": { + "auditee_uei": "LJNSJ1C1KU15" + } + } +] \ No newline at end of file diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx new file mode 100644 index 0000000000..2cb89768d1 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx new file mode 100644 index 0000000000..90e168e3a3 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx new file mode 100644 index 0000000000..8376edf4df Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx new file mode 100644 index 0000000000..1fa5ed06af Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx new file mode 100644 index 0000000000..9f9e80ef1e Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx new file mode 100644 index 0000000000..627bf95d68 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx new file mode 100644 index 0000000000..6a4d10f9b5 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx new file mode 100644 index 0000000000..5434f8e036 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/test-array-180818.json b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/test-array-180818.json new file mode 100644 index 0000000000..ed7651efd0 --- /dev/null +++ b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/test-array-180818.json @@ -0,0 +1,1519 @@ +[ + { + "endpoint": "additional_eins", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": { + "auditee_uei": "XQGMGEP47M41" + } + }, + { + "endpoint": "additional_ueis", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": { + "auditee_uei": "XQGMGEP47M41" + } + }, + { + "endpoint": "corrective_action_plans", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": { + "auditee_uei": "XQGMGEP47M41" + } + }, + { + "endpoint": "federal_awards", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [ + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "553", + "SCHOOL BREAKFAST PROGRAM", + "nan", + 993118, + "nan", + 4312571, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 993118, + "10", + "553", + "AWARD-0001", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "011", + "MIGRANT EDUCATION_STATE GRANT PROGRAM", + "nan", + 281211, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 281211, + "84", + "011", + "AWARD-0002", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "SPECIAL EDUCATION_GRANTS TO STATES", + "nan", + 2186370, + "nan", + 2254274, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 670133, + "84", + "027", + "AWARD-0003", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "SPECIAL EDUCATION_GRANTS TO STATES", + "nan", + 2186370, + "nan", + 2254274, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 1184601, + "84", + "027", + "AWARD-0004", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "SPECIAL EDUCATION_GRANTS TO STATES", + "nan", + 2186370, + "nan", + 2254274, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 21171, + "84", + "027", + "AWARD-0005", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "027", + "SPECIAL EDUCATION_GRANTS TO STATES", + "nan", + 2186370, + "COVID-19", + 2254274, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 310465, + "84", + "027", + "AWARD-0006", + "COVID-19", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "SPECIAL EDUCATION_PRESCHOOL GRANTS", + "nan", + 67904, + "nan", + 2254274, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 46657, + "84", + "173", + "AWARD-0007", + "", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "173", + "SPECIAL EDUCATION_PRESCHOOL GRANTS", + "nan", + 67904, + "COVID-19", + 2254274, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 21247, + "84", + "173", + "AWARD-0008", + "COVID-19", + "SPECIAL EDUCATION CLUSTER (IDEA)" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "048", + "CAREER AND TECHNICAL EDUCATION -- BASIC GRANTS TO STATES", + "nan", + 81725, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 81725, + "84", + "048", + "AWARD-0009", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "181", + "SPECIAL EDUCATION-GRANTS FOR INFANTS AND FAMILIES", + "nan", + 56956, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 56956, + "84", + "181", + "AWARD-0010", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "287", + "TWENTY-FIRST CENTURY COMMUNITY LEARNING CENTERS", + "nan", + 330807, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 330807, + "84", + "287", + "AWARD-0011", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "555", + "NATIONAL SCHOOL LUNCH PROGRAM", + "nan", + 3118836, + "nan", + 4312571, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 2767881, + "10", + "555", + "AWARD-0012", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "365", + "ENGLISH LANGUAGE ACQUISITION STATE GRANTS", + "nan", + 37878, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 37878, + "84", + "365", + "AWARD-0013", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "367", + "IMPROVING TEACHER QUALITY STATE GRANTS", + "nan", + 225483, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 225483, + "84", + "367", + "AWARD-0014", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "371", + "STRIVING READERS", + "nan", + 25446, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 25446, + "84", + "371", + "AWARD-0015", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "424", + "STUDENT SUPPORT AND ACADEMIC ENRICHMENT PROGRAM", + "nan", + 65945, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 65945, + "84", + "424", + "AWARD-0016", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "EDUCATION STABILIZATION FUND", + "nan", + 4348939, + "COVID-19, 84.425C", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 0, + 115727, + "84", + "425", + "AWARD-0017", + "COVID-19, 84.425C", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "EDUCATION STABILIZATION FUND", + "nan", + 4348939, + "COVID-19, 84.425D", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 0, + 2973187, + "84", + "425", + "AWARD-0018", + "COVID-19, 84.425D", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "425", + "EDUCATION STABILIZATION FUND", + "nan", + 4348939, + "COVID-19, 84.425U", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 0, + 1260025, + "84", + "425", + "AWARD-0019", + "COVID-19, 84.425U", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "575", + "CHILD CARE AND DEVELOPMENT BLOCK GRANT", + "nan", + 31966, + "nan", + 31966, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 31966, + "93", + "575", + "AWARD-0020", + "", + "CCDF CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "434", + "EVERY STUDENT SUCCEEDS ACT/PRESCHOOL DEVELOPMENT GRANTS", + "nan", + 18503, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 18503, + "93", + "434", + "AWARD-0021", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "590", + "COMMUNITY-BASED CHILD ABUSE PREVENTION GRANTS", + "nan", + 30000, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 30000, + "93", + "590", + "AWARD-0022", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "555", + "NATIONAL SCHOOL LUNCH PROGRAM", + "nan", + 3118836, + "nan", + 4312571, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 220526, + "10", + "555", + "AWARD-0023", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "555", + "NATIONAL SCHOOL LUNCH PROGRAM", + "nan", + 3118836, + "COVID-19", + 4312571, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 64157, + "10", + "555", + "AWARD-0024", + "COVID-19", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "555", + "NATIONAL SCHOOL LUNCH PROGRAM", + "nan", + 3118836, + "COVID-19", + 4312571, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 66272, + "10", + "555", + "AWARD-0025", + "COVID-19", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "559", + "SUMMER FOOD SERVICE PROGRAM FOR CHILDREN", + "nan", + 200617, + "nan", + 4312571, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 200617, + "10", + "559", + "AWARD-0026", + "", + "CHILD NUTRITION CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "579", + "CHILD NUTRITION DISCRETIONARY GRANTS LIMITED AVAILABILITY", + "nan", + 8905, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 8905, + "10", + "579", + "AWARD-0027", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "21", + "027", + "CORONAVIRUS STATE AND LOCAL FISCAL RECOVERY FUNDS", + "nan", + 348802, + "COVID-19", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 348802, + "21", + "027", + "AWARD-0028", + "COVID-19", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "84", + "010", + "TITLE I GRANTS TO LOCAL EDUCATIONAL AGENCIES", + "nan", + 909648, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 0, + 909648, + "84", + "010", + "AWARD-0029", + "", + "N/A" + ] + } + ], + "singletons": { + "auditee_uei": "XQGMGEP47M41", + "total_amount_expended": 13369059 + } + }, + { + "endpoint": "findings_text", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": { + "auditee_uei": "XQGMGEP47M41" + } + }, + { + "endpoint": "findings", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": {} + }, + { + "endpoint": "notes_to_sefa", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": { + "accounting_policies": "The accompanying schedule of expenditures of federal awards (the Schedule) includes the federal award activity of the Board of Education of Queen Annes County, Maryland under programs of the federal government for the year ended June 30, 2022.The information in this Schedule is presented in accordance with the requirements of Title 2 U.S. Code of Federal Regulations Part 200, Uniform Administrative Requirements, Cost Principles, and Audit Requirements for Federal Awards (Uniform Guidance).Because the Schedule presents only a selected portion of the operations of the Board of Education of Queen Annes County, Maryland it is not intended to and does not present the financial position, changes in net assets, or cash flows of the Board of Education of Queen Annes County, Maryland.Expenditures reported on the Schedule are reported on the modified accrual basis of accounting.Such expenditures are recognized following the cost principles contained in the Uniform Guidance, wherein certain types of expenditures are not allowable or are limited as to reimbursement.", + "auditee_uei": "XQGMGEP47M41", + "is_minimis_rate_used": "N", + "rate_explained": "The auditee did not use the de minimis cost rate." + } + }, + { + "endpoint": "secondary_auditor", + "report_id": "2022-06-CENSUS-0000180818", + "rows": [], + "singletons": { + "auditee_uei": "XQGMGEP47M41" + } + } +] \ No newline at end of file diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx new file mode 100644 index 0000000000..d4265e3441 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx new file mode 100644 index 0000000000..a79650b8c3 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx new file mode 100644 index 0000000000..b091e05e30 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx new file mode 100644 index 0000000000..6543b9606d Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx new file mode 100644 index 0000000000..ea927dc6f5 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx new file mode 100644 index 0000000000..3cff55b833 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx new file mode 100644 index 0000000000..3aa84aa820 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx new file mode 100644 index 0000000000..4bee2563e7 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/test-array-251020.json b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/test-array-251020.json new file mode 100644 index 0000000000..efe4f20657 --- /dev/null +++ b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/test-array-251020.json @@ -0,0 +1,395 @@ +[ + { + "endpoint": "additional_eins", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [], + "singletons": { + "auditee_uei": "GM27HGUL61B5" + } + }, + { + "endpoint": "additional_ueis", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [], + "singletons": { + "auditee_uei": "GM27HGUL61B5" + } + }, + { + "endpoint": "corrective_action_plans", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [ + { + "fields": [ + "finding_ref_number", + "planned_action", + "contains_chart_or_table" + ], + "values": [ + "2022-004", + "Finding 2022-004 Significant Deficiency Assistance Listing:21.027 Coronavirus State and Local Fiscal Recovery Funds (CSLFRF)Federal Grantor:Department of the Treasury Compliance Requirement:Procurement and Suspension and DebarmentCondition:The District used a sole source authorization for the procurement of a Type-1 Fire Engine but a competitive bid process should have been used to comply with Uniform Guidance.Recommendation:We recommend the District work with FEMA to obtain written approval for the sole source procurement, which is one of the exceptions to noncompetitive procurements.Management Response and Corrective Action Plan: The District shall revise policies and procedures to incorporate the requirements in the Uniform Guidance in its sole source approval process when it comes to selecting and approving vendors for expenditures that relates to a federal grant. The District will also work with the awarding agency to ensure written approval are obtained for sole source purchases.", + "N" + ] + }, + { + "fields": [ + "finding_ref_number", + "planned_action", + "contains_chart_or_table" + ], + "values": [ + "2022-002", + "Finding 2022-002 Significant Deficiency Award No.:97.083, Staffing for Adequate Fire and Emergency ResponseFederal Grantor:U.S.Department of Homeland Security, Federal Emergency Management AgencyCompliance Requirement:Other compliance requirements.Condition:The schedule of Expenditures of Federal Awards (SEFA) was not complete, and expenditures reported on the SEFA were revised during the single audit.Recommendation:We recommend additional review procedures be implemented to ensure the SEFA is complete and accurate when the single audit begins.Management Response and Corrective Action Plan: The Finance division will work with other departments to ensure that data provided in the SEFA are complete and accounted for on an accrual basis. We will also implement efficiencies in our accounting systems to ensure expenditures are captured correctly to prevent errors and omissions. Additional review will be completed by the Finance Director for completeness.", + "N" + ] + }, + { + "fields": [ + "finding_ref_number", + "planned_action", + "contains_chart_or_table" + ], + "values": [ + "2022-003", + "Finding 2022-003 Significant Deficiency Assistance Listing:21.027 Coronavirus State and Local Fiscal Recovery Funds (CSLFRF)Federal Grantor:Department of the TreasuryCompliance Requirement:ReportingCondition:The Project and Expenditure Reports were not filed.Recommendation:We recommend the District file the initial Project and Expenditures Report for the period covering March 3, 2021 to March 31,2022 as soon as possible.Subsequent annual reports should be filed by the April 30, 2023 deadline.Management Response and Corrective Action Plan: The District has confirmed with the City of Elk Grove that as the main recipient of the grant, the City has filed the project and expenditure reports with the Treasury Department. In addition to what has already been report, the District will establish the proper authority to report the project and expenditure reports to the Treasury Department for period covering March 3, 2021 to March 31, 2022. The District will ensure that going forward projects and expenditures are reported in accordance with the schedule set forth by the guidance issued by the Treasury.", + "N" + ] + } + ], + "singletons": { + "auditee_uei": "GM27HGUL61B5" + } + }, + { + "endpoint": "federal_awards", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [ + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "97", + "083", + "STAFFING FOR ADEQUATE FIRE AND EMERGENCY RESPONSE (SAFER)", + "nan", + 2564216, + "nan", + 0, + "N", + "nan", + "Y", + "N", + "", + "N", + "nan", + 1, + 2564216, + "97", + "083", + "AWARD-0001", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "21", + "027", + "CORONAVIRUS STATE AND LOCAL FISCAL RECOVERY FUNDS", + "nan", + 1458378, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "Y", + "U", + 2, + 1458378, + "21", + "027", + "AWARD-0002", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "97", + "044", + "ASSISTANCE TO FIREFIGHTERS GRANT", + "nan", + 289749, + "nan", + 0, + "N", + "nan", + "Y", + "N", + "", + "Y", + "U", + 0, + 289749, + "97", + "044", + "AWARD-0003", + "", + "N/A" + ] + } + ], + "singletons": { + "auditee_uei": "GM27HGUL61B5", + "total_amount_expended": 4312343 + } + }, + { + "endpoint": "findings_text", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [ + { + "fields": [ + "finding_ref_number", + "finding_text", + "contains_chart_or_table" + ], + "values": [ + "2022-003", + "Finding 2022-003 Significant Deficiency Assistance Listing:21.027 Coronavirus State and Local Fiscal Recovery Funds (CSLFRF)Federal Grantor:Department of the TreasuryCompliance Requirement:ReportingCondition:The Project and Expenditure Reports were not filed.Criteria:The Subrecipient Agreement between the City of Elk Grove and the District. Section 10 states Subrecipient shall comply with any applicable reporting obligations established by the Department of the Treasury as they relate to this Agreement.Department of Treasury Compliance and Reporting Guidance for CSLFRF, Part 2, Section B outlines reporting requirements.Reports on projects funded, expenditures, and contracts and subawards equal to or greater than $50,000, and other information are required annually for recipients serving populations under 250,000 and receiving less than $10 million in CSLFRF funding.Cause:The District believed that as a subrecipient of funds the funding provider would prepare and file the required reporting.Effect:Treasury may impose additional conditions on the receipt of a subsequent tranche of future award funds, if any, or take other available remedies as set forth in 2 C.F.R. 200.339.Recommendation:We recommend the District file the initial Project and Expenditures Report for the period covering March 3, 2021 to March 31,2022 as soon as possible.Subsequent annual reports should be filed by the April 30, 2023 deadline.Views of Responsible Officials and Planned Corrective Actions:Managements response and planned corrective action is included in the Corrective Action Plan included at the end of the report.", + "N" + ] + }, + { + "fields": [ + "finding_ref_number", + "finding_text", + "contains_chart_or_table" + ], + "values": [ + "2022-004", + "Finding 2022-004 Significant Deficiency Assistance Listing:21.027 Coronavirus State and Local Fiscal Recovery Funds (CSLFRF)Federal Grantor:Department of the Treasury Compliance Requirement:Procurement and Suspension and DebarmentCondition:The District used a sole source authorization for the procurement of a Type-1 Fire Engine but a competitive bid process should have been used to comply with Uniform Guidance.Criteria:Non-state entities under the program, must follow the procurement standards in 2 CFR sections 200.318 through 200.327, including ensuring that the procurement method used for the contracts are appropriate based on the dollar amount and conditions specified in 2 CFR section 200.320.Specifically, procurements in excess of $250,000 be conducted using a competitive procurement method such as bids or proposals. Further, the Districts Purchasing Policy Sec 2112.13 requires staff obtain competitive bids for the purchase of supplies, equipment or service that are $15,000 or greater.Interagency or cooperative purchasing agreements may be leveraged, but they must comply with the Unform Standards, and 2 CFR section 200.320(c) states noncompetitive procurement can only be awarded if one or more of the following circumstances apply:1)The acquisition of property or services, the aggregate dollar amount of which does not exceed the micro-purchase threshold (see paragraph (a)(1) of this section);2)The item is available only from a single source;3)The public exigency or emergency for the requirement will not permit a delay resulting from publicizing a competitive solicitation;4)The Federal awarding agency or pass-through entity expressly authorizes a noncompetitive procurement in response to a written request from the non-Federal entity; or5)After solicitation of a number of sources, competition is determined inadequate.Cause:The Districts Purchasing Policy allows for a sole source vendor with the approval of the Chief Administrative Officer.In addition, the Policy allows the District to purchase through a purchasing alliance, such as a cooperative agreement.The sole source vendor has a cooperative purchasing agreement through National Purchasing Partners (NPP) cooperative, contract number PS20240.The Districts justification for sole source vendor was uniformity of vendors across safety equipment which fosters ease of operator training and emergency scene efficiency.The approval indicated standardization also supports ongoing maintenance of the fleet.While permitted by internal policy, this justification does not appear to meet any of the noncompetitive procurement exceptions allowed by the Uniform Standards outlined above.The District was also unable to provide documentation that the vendor purchasing agreement through NPP complies with the Uniform Standards for procurement.Effect:Treasury may impose additional conditions on the receipt of a subsequent tranche of future award funds, if any, or take other available remedies as set forth in 2 C.F.R. 200.339.Recommendation:We recommend the District work with FEMA to obtain written approval for the sole source procurement, which is one of the exceptions to noncompetitive procurements.Views of Responsible Officials and Planned Corrective Actions:Managements response and planned corrective action is included in the Corrective Action Plan included at the end of the report.", + "N" + ] + }, + { + "fields": [ + "finding_ref_number", + "finding_text", + "contains_chart_or_table" + ], + "values": [ + "2022-002", + "Finding 2022-002 Significant Deficiency Award No.:97.083, Staffing for Adequate Fire and Emergency ResponseFederal Grantor:U.S.Department of Homeland Security, Federal Emergency Management AgencyCompliance Requirement:Other compliance requirements.Condition:The schedule of Expenditures of Federal Awards (SEFA) was not complete, and expenditures reported on the SEFA were revised during the single audit.Criteria:2 CFR Part 200, Subpart F (Uniform Guidance) Section 200.502 states, The auditee should prepare a Schedule of Expenditures of Federal Awards for the period covered by the auditees financial statements.Internal controls over the SEFA should be in place ensure accrual basis expenses incurred under the federal program are properly reported as expenses on the SEFA and are properly reported as revenue in the financial statements prior to the start of the single audit.Cause:SEFA was not fully reconciled and finalized until after the single audit began.Effect:The expenses included on the SEFA were revised during the single audit, which could have resulted in the auditor not selecting the correct major program or expenses for testing and could have resulted in the single audit not satisfying the requirements of the Uniform Guidance.Recommendation:We recommend additional review procedures be implemented to ensure the SEFA is complete and accurate when the single audit begins.Views of Responsible Officials and Planned Corrective Actions:Managements response and planned corrective action is included in the Corrective Action Plan included at the end of the report.", + "N" + ] + } + ], + "singletons": { + "auditee_uei": "GM27HGUL61B5" + } + }, + { + "endpoint": "findings", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [ + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "L", + "2022-003", + "N", + "Y", + "N", + "Y", + "N", + "N", + "N", + "AWARD-0002" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "I", + "2022-004", + "N", + "Y", + "N", + "Y", + "N", + "N", + "N", + "AWARD-0002" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "prior_finding_ref_numbers", + "award_reference" + ], + "values": [ + "P", + "2022-002", + "N", + "N", + "N", + "Y", + "N", + "N", + "Y", + "2021-003", + "AWARD-0001" + ] + } + ], + "singletons": {} + }, + { + "endpoint": "notes_to_sefa", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [ + { + "fields": [ + "title", + "content" + ], + "values": [ + "Basis of Presentation", + "The accompanying schedule of expenditures of federal awards includes the federal grant activity of the Cosumnes Community Services District (the District) under programs of the federal government for the year ended June 30, 2021. The information in this schedule is presented in accordance with the requirements of Title 2 U.S. Code of Federal Regulations Part 200, Uniform Administrative Requirements, Cost Principles, and Audit Requirements for Federal Awards (Uniform Guidance). Because the Schedule presents only a selected portion of the Districts operations, it is not intended to be and does not present the financial position, changes in net position, or cash flows of the District. Therefore, some amounts presented in this schedule may differ from amounts presented in, or used in the preparation of, the basic financial statements." + ] + }, + { + "fields": [ + "title", + "content" + ], + "values": [ + "Subrecipients", + "The District did not have any subrecipients of its federal programs." + ] + } + ], + "singletons": { + "accounting_policies": "Expenses reported on the Schedule are reported on the accrual basis of accounting. Such expenses are recognized following the cost principles contained in the Uniform Guidance, wherein certain types of expenses are not allowable or are limited as to reimbursement.The amounts passed through to subrecipients, if any, are reported on in this schedule when disbursed in accordance with 2 CFR 200.502(a), which differs from the accrual basis of accounting used under generally accepted accounting principles.", + "auditee_uei": "GM27HGUL61B5", + "is_minimis_rate_used": "Both", + "rate_explained": "The District did not charge indirect costs to its federal programs." + } + }, + { + "endpoint": "secondary_auditor", + "report_id": "2022-06-CENSUS-0000251020", + "rows": [], + "singletons": { + "auditee_uei": "GM27HGUL61B5" + } + } +] \ No newline at end of file diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx new file mode 100644 index 0000000000..539698e48e Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx new file mode 100644 index 0000000000..6e239b6a9c Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx new file mode 100644 index 0000000000..1732a0defc Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx new file mode 100644 index 0000000000..47c5e722b3 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx new file mode 100644 index 0000000000..3bdf6543eb Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx new file mode 100644 index 0000000000..904c2ea11c Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx new file mode 100644 index 0000000000..50c53597e7 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx new file mode 100644 index 0000000000..2931db4874 Binary files /dev/null and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/test-array-69688.json b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/test-array-69688.json new file mode 100644 index 0000000000..3f6458d984 --- /dev/null +++ b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/test-array-69688.json @@ -0,0 +1,622 @@ +[ + { + "endpoint": "additional_eins", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [], + "singletons": { + "auditee_uei": "LBNNLRDE8793" + } + }, + { + "endpoint": "additional_ueis", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [], + "singletons": { + "auditee_uei": "LBNNLRDE8793" + } + }, + { + "endpoint": "corrective_action_plans", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [ + { + "fields": [ + "finding_ref_number", + "planned_action", + "contains_chart_or_table" + ], + "values": [ + "2022-001", + "CORRECTIVE ACTION PLANManagement will adopt an internal control process that will alert the Health Center when reporting due dates are approaching.", + "N" + ] + }, + { + "fields": [ + "finding_ref_number", + "planned_action", + "contains_chart_or_table" + ], + "values": [ + "2022-002", + "CORRECTIVE ACTION PLANManagement has developed the following for consistent adjustments to patient accounts according to the SFDS:1. Management will work with the electronic payment system contractors to update system parameters for automatic system generated discounts in accordance with the sliding discount schedule. This process began February 28, 2023.2. Management will implement a monthly audit of a statistically relevant sample of all encounters subjected to the sliding fee adjustment process to test the consistency of the adjustment with the SFDS. This process began February 28, 2023.3. Management will implement a process to ensure that all reviews and audit corrections are performed by a person other than the person performing the review and all adjustments to patient accounts are reviewed subsequent to processing. Anticipated completion date is June 30, 2023.", + "N" + ] + } + ], + "singletons": { + "auditee_uei": "LBNNLRDE8793" + } + }, + { + "endpoint": "federal_awards", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [ + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "10", + "557", + "SPECIAL SUPPLEMENTAL NUTRITION PROGRAM FOR WOMEN, INFANTS, AND CHILDREN", + "nan", + 512929, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 512929, + "10", + "557", + "AWARD-0001", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "224", + "CONSOLIDATED HEALTH CENTERS (COMMUNITY HEALTH CENTERS, MIGRANT HEALTH CENTERS, HEALTH CARE FOR THE HOMELESS, AND PUBLIC HOUSING PRIMARY CARE)", + "nan", + 5973065, + "nan", + 6420692, + "N", + "nan", + "Y", + "N", + "", + "Y", + "U", + 2, + 4485556, + "93", + "224", + "AWARD-0002", + "", + "HEALTH CENTER PROGRAM CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "527", + "AFFORDABLE CARE ACT (ACA) GRANTS FOR NEW AND EXPANDED SERVICES UNDER THE HEALTH CENTER PROGRAM", + "nan", + 447627, + "nan", + 6420692, + "N", + "nan", + "Y", + "N", + "", + "Y", + "U", + 0, + 447627, + "93", + "527", + "AWARD-0003", + "", + "HEALTH CENTER PROGRAM CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "224", + "COVID-19 - CONSOLIDATED HEALTH CENTERS (COMMUNITY HEALTH CENTERS, MIGRANT HEALTH CENTERS, HEALTH CARE FOR THE HOMELESS, AND PUBLIC HOUSING PRIMARY CARE)", + "nan", + 5973065, + "COVID-19", + 6420692, + "N", + "nan", + "Y", + "N", + "", + "Y", + "U", + 0, + 1487509, + "93", + "224", + "AWARD-0004", + "COVID-19", + "HEALTH CENTER PROGRAM CLUSTER" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "926", + "HEALTHY START INITIATIVE", + "nan", + 826156, + "nan", + 0, + "N", + "nan", + "Y", + "N", + "", + "N", + "nan", + 0, + 826156, + "93", + "926", + "AWARD-0005", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "889", + "NATIONAL BIOTERRORISM HOSPITAL PREPAREDNESS PROGRAM", + "nan", + 4996, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 4996, + "93", + "889", + "AWARD-0006", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "940", + "HIV PREVENTION ACTIVITIES_HEALTH DEPARTMENT BASED", + "nan", + 20001, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 20001, + "93", + "940", + "AWARD-0007", + "", + "N/A" + ] + }, + { + "fields": [ + "federal_agency_prefix", + "three_digit_extension", + "federal_program_name", + "state_cluster_name", + "federal_program_total", + "additional_award_identification", + "cluster_total", + "is_loan", + "loan_balance", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "findings_count", + "amount_expended", + "federal_agency_prefix", + "three_digit_extension", + "award_reference", + "additional_award_identification", + "cluster_name", + "other_cluster_name", + "" + ], + "values": [ + "93", + "217", + "FAMILY PLANNING_SERVICES", + "nan", + 290916, + "nan", + 0, + "N", + "nan", + "N", + "N", + "", + "N", + "nan", + 0, + 290916, + "93", + "217", + "AWARD-0008", + "", + "N/A" + ] + } + ], + "singletons": { + "auditee_uei": "LBNNLRDE8793", + "total_amount_expended": 8075690 + } + }, + { + "endpoint": "findings_text", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [ + { + "fields": [ + "finding_ref_number", + "finding_text", + "contains_chart_or_table" + ], + "values": [ + "2022-002", + "Health Center Program ClusterAssistance Listing Number(s) 93.224 and 93.527U.S. Department of Health and Human ServicesAward No. H80CS00438Criteria or Specific Requirement: Special Tests and ProvisionsType of Finding: Significant Deficiency in Internal Control Over ComplianceCondition: The Health Center must prepare and apply a sliding fee discount schedule (SFDS) so that the amounts owed for health center services by eligible participants are adjusted based on the patients ability to pay. Questioned costs: n/aContext: The auditor selected a sample of 60 encounters. After the auditor tested selection #49, enough errors had been identified that the auditor stopped testing because the sample results could not support a low assessed level of control risk. The errors were a result of the misapplication of the sliding fee discount. The sliding fee discounts are based on the slide level code. In these instances, the sliding fee discount that was applied was not in accordance with the SFDS. Effect or Potential Effect: Either the Health Center or the patients may be paying more than required under the contract.Cause: The Health Center applied its SFDS inconsistently to the patients selected for testing. Recommendation: The Health Center should adopt an internal control procedure that allows them to consistently apply its SFDS to all eligible patients. Repeat Finding: 2021-001Views of Responsible Officials: See page 12.", + "N" + ] + }, + { + "fields": [ + "finding_ref_number", + "finding_text", + "contains_chart_or_table" + ], + "values": [ + "2022-001", + "Health Center Program ClusterAssistance Listing Number(s) 93.224 and 93.527U.S. Department of Health and Human ServicesCriteria or Specific Requirement: ReportingType of Finding: Significant Deficiency in Internal Control Over ComplianceCondition: The Health Center must timely file its SF-425 Federal Financial Reports for each of its awards.Questioned costs: n/aContext: The auditor found that some of the Health Centers Forms SF-425 were filed after their respective due dates.Effect or Potential Effect: Information about the awards is not communicated timely to the awarding agency. Cause: The Health Center had inadequate controls that would prevent them from missing reporting due dates. Recommendation: The Health Center should adopt an internal control procedure that allows them to consistently file its reports on a timely basis. Repeat Finding: n/aViews of Responsible Officials: See page 12.", + "N" + ] + } + ], + "singletons": { + "auditee_uei": "LBNNLRDE8793" + } + }, + { + "endpoint": "findings", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [ + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "prior_finding_ref_numbers", + "award_reference" + ], + "values": [ + "N", + "2022-002", + "N", + "N", + "N", + "Y", + "N", + "N", + "Y", + "2021-001", + "AWARD-0003" + ] + }, + { + "fields": [ + "type_requirement", + "reference_number", + "is_modified_opinion", + "is_other_matters", + "is_material_weakness", + "is_significant_deficiency", + "is_other_findings", + "is_questioned_costs", + "is_repeat_finding", + "award_reference" + ], + "values": [ + "L", + "2022-001", + "N", + "N", + "N", + "Y", + "N", + "N", + "N", + "AWARD-0003" + ] + } + ], + "singletons": {} + }, + { + "endpoint": "notes_to_sefa", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [ + { + "fields": [ + "title", + "content" + ], + "values": [ + "Basis of Presentation", + "The accompanying Schedule of Expenditures of Federal Awards (the \"Schedule\") includes the federal award activity of Charles Drew Health Center, Inc. under programs of the federal government for the year ended August 31, 2022. The information in this Schedule is presented in accordance with the requirements of Title 2 U.S. Code of Federal Regulations Part 200, Uniform Administrative Requirements, Cost Principles, and Audit Requirements for Federal Awards (Uniform Guidance). Because the Schedule presents only a selected portion of the operations of Charles Drew Health Center, Inc., it is not intended to and does not present the financial position, changes in net assets or cash flows of Charles Drew Health Center, Inc." + ] + }, + { + "fields": [ + "title", + "content" + ], + "values": [ + "Contingencies", + "Charles Drew Health Center, Inc. receives funds under various federal grant programs, and such assistance is to be expended in accordance with the provisions of the various grants. Compliance with the grants is subject to audit by various government agencies which may impose sanctions in the event of noncompliance. Management believes that they have complied with all material aspects of the various grant provisions and the results of adjustments, if any, relating to such audits would not have any material financial impacts." + ] + } + ], + "singletons": { + "accounting_policies": "Expenditures reported on the Schedule are reported on the accrual basis of accounting. Such expenditures are recognized following the cost principles contained in the Uniform Guidance, wherein certain types of expenditures are not allowable or are limited as to reimbursement. Pass-through entity identifying numbers are presented where available.", + "auditee_uei": "LBNNLRDE8793", + "is_minimis_rate_used": "Y", + "rate_explained": "The auditee used the de minimis cost rate. Charles Drew Health Center, Inc. has elected to use the 10% de minimis indirect cost rate allowed under the Uniform Guidance." + } + }, + { + "endpoint": "secondary_auditor", + "report_id": "2022-08-CENSUS-0000069688", + "rows": [ + { + "fields": [ + "address_city", + "contact_name", + "auditor_ein", + "contact_email", + "auditor_name", + "contact_phone", + "address_state", + "address_street", + "contact_title", + "address_zipcode" + ], + "values": [ + "OMAHA", + "KATIE BYRD", + "470625816", + "KBYRD@LUTZ.US", + "LUTZ & COMPANY PC", + "4024968800", + "NE", + "13616 CALIFORNIA STREET, SUITE 300", + "DIRECTOR", + "68154" + ] + } + ], + "singletons": { + "auditee_uei": "LBNNLRDE8793" + } + } +] \ No newline at end of file diff --git a/backend/census_historical_migration/historic_data_loader.py b/backend/census_historical_migration/historic_data_loader.py new file mode 100644 index 0000000000..dafc93c965 --- /dev/null +++ b/backend/census_historical_migration/historic_data_loader.py @@ -0,0 +1,59 @@ +from .models import ELECAUDITHEADER as Gen +from .workbooklib.end_to_end_core import run_end_to_end + +from django.contrib.auth import get_user_model + +User = get_user_model() + + +def load_historic_data_for_year(audit_year): + """Iterates over and processes submissions for the given audit year""" + result_log = {} + total_count = error_count = 0 + user = create_or_get_user() + submissions_for_year = Gen.objects.filter(AUDITYEAR=audit_year) + + for submission in submissions_for_year: + dbkey = submission.DBKEY + result = {"success": [], "errors": []} + + try: + # Migrate a single submission + run_end_to_end(user, dbkey, audit_year, result) + except Exception as exc: + result["errors"].append(f"{exc}") + + result_log[(audit_year, dbkey)] = result + total_count += 1 + + if len(result["errors"]) > 0: + error_count += 1 + if total_count % 5 == 0: + print(f"Processed = {total_count}, Errors = {error_count}") + if error_count > 5: + break + + print("********* Loader Summary ***************") + + for k, v in result_log.items(): + print(k, v) + print("-------------------") + + print(f"{error_count} errors out of {total_count}") + + +def create_or_get_user(): + """Returns the default migration user""" + user_email = "fac-census-migration-auditee-official@fac.gsa.gov" + user_name = "fac-census-migration-auditee-official" + user = None + + users = User.objects.filter(email=user_email) + if users: + user = users.first() + else: + print("Creating user", user_email, user_name) + user = User(username=user_name, email=user_email) + user.save() + + return user diff --git a/backend/census_historical_migration/management/commands/historic_data_migrator.py b/backend/census_historical_migration/management/commands/historic_data_migrator.py index 352d9b348c..74d0b1ca6f 100644 --- a/backend/census_historical_migration/management/commands/historic_data_migrator.py +++ b/backend/census_historical_migration/management/commands/historic_data_migrator.py @@ -1,18 +1,16 @@ -import os import logging import sys +from census_historical_migration.historic_data_loader import create_or_get_user from config.settings import ENVIRONMENT from django.core.management.base import BaseCommand from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end -CYPRESS_TEST_EMAIL_ADDR = os.getenv("CYPRESS_LOGIN_TEST_EMAIL_AUDITEE") logger = logging.getLogger(__name__) class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument("--email", type=str, required=False) parser.add_argument("--dbkeys", type=str, required=False, default="") parser.add_argument("--years", type=str, required=False, default="") @@ -35,12 +33,11 @@ def handle(self, *args, **options): logger.error("Years must be two digits. Exiting.") sys.exit(-2) - email = options.get("email", CYPRESS_TEST_EMAIL_ADDR) + user = create_or_get_user() defaults = [ - (182926, 22), - (181744, 22), - (191734, 22), + (177310, 22), + (251020, 22), ] if ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]: @@ -49,11 +46,15 @@ def handle(self, *args, **options): f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" ) for dbkey, year in zip(dbkeys, years): - run_end_to_end(email, dbkey, year) + result = {"success": [], "errors": []} + run_end_to_end(user, dbkey, year, result) + logger.info(result) else: for pair in defaults: logger.info("Running {}-{} end-to-end".format(pair[0], pair[1])) - run_end_to_end(email, str(pair[0]), str(pair[1])) + result = {"success": [], "errors": []} + run_end_to_end(user, str(pair[0]), str(pair[1]), result) + logger.info(result) else: logger.error( "Cannot run end-to-end workbook generation in production. Exiting." diff --git a/backend/census_historical_migration/management/commands/run_migration_for_year.py b/backend/census_historical_migration/management/commands/run_migration_for_year.py new file mode 100644 index 0000000000..63f799c061 --- /dev/null +++ b/backend/census_historical_migration/management/commands/run_migration_for_year.py @@ -0,0 +1,28 @@ +from ...historic_data_loader import load_historic_data_for_year + +from django.core.management.base import BaseCommand + +import logging + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) + + +class Command(BaseCommand): + help = """ + Migrate from Census tables to GSAFAC tables for a given year + Usage: + manage.py run_migration --year + """ + + def add_arguments(self, parser): + parser.add_argument("--year", help="4-digit Audit Year") + + def handle(self, *args, **options): + year = options.get("year") + if not year: + print("Please specify an audit year") + return + + load_historic_data_for_year(audit_year=year) diff --git a/backend/census_historical_migration/migrations/0002_reportmigrationstatus_migrationerrordetail.py b/backend/census_historical_migration/migrations/0002_reportmigrationstatus_migrationerrordetail.py new file mode 100644 index 0000000000..6179d55c43 --- /dev/null +++ b/backend/census_historical_migration/migrations/0002_reportmigrationstatus_migrationerrordetail.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.6 on 2023-11-27 19:36 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("census_historical_migration", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="ReportMigrationStatus", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("audit_year", models.TextField(blank=True, null=True)), + ("dbkey", models.TextField(blank=True, null=True)), + ( + "run_datetime", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("migration_status", models.TextField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name="MigrationErrorDetail", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("error_summary", models.CharField(blank=True, null=True)), + ("error_stack", models.TextField()), + ( + "report_migration_status", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="census_historical_migration.reportmigrationstatus", + ), + ), + ], + ), + ] diff --git a/backend/census_historical_migration/models.py b/backend/census_historical_migration/models.py index 503a9e027f..99e3502b42 100644 --- a/backend/census_historical_migration/models.py +++ b/backend/census_historical_migration/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.utils import timezone class ELECAUDITHEADER(models.Model): @@ -443,3 +444,18 @@ class ELECCAPTEXT(models.Model): UEI = models.TextField(blank=True, null=True) MULTIPLEUEIS = models.TextField(blank=True, null=True) + + +class ReportMigrationStatus(models.Model): + audit_year = models.TextField(blank=True, null=True) + dbkey = models.TextField(blank=True, null=True) + run_datetime = models.DateTimeField(default=timezone.now) + migration_status = models.TextField(blank=True, null=True) + + +class MigrationErrorDetail(models.Model): + report_migration_status = models.ForeignKey( + ReportMigrationStatus, on_delete=models.CASCADE + ) + error_summary = models.CharField(blank=True, null=True) + error_stack = models.TextField() diff --git a/backend/census_historical_migration/sac_general_lib/report_id_generator.py b/backend/census_historical_migration/sac_general_lib/report_id_generator.py index dbefce58c9..80c581329d 100644 --- a/backend/census_historical_migration/sac_general_lib/report_id_generator.py +++ b/backend/census_historical_migration/sac_general_lib/report_id_generator.py @@ -1,8 +1,13 @@ -from census_historical_migration.sac_general_lib.utils import _census_date_to_datetime +from census_historical_migration.sac_general_lib.utils import ( + _census_date_to_datetime, + xform_census_date_to_datetime, +) # FIXME: Get the padding/shape right on the report_id -def dbkey_to_report_id(Gen, dbkey): +def dbkey_to_report_id( + Gen, dbkey +): # FIXME-MSHD: Remove this function once we switch all workbook generators to using census models g = Gen.select(Gen.audityear, Gen.fyenddate).where(Gen.dbkey == dbkey).get() # month = g.fyenddate.split('-')[1] # 2022JUN0001000003 @@ -11,3 +16,13 @@ def dbkey_to_report_id(Gen, dbkey): # historic DBKEY report_ids dt = _census_date_to_datetime(g.fyenddate) return f"{g.audityear}-{dt.month:02}-CENSUS-{dbkey.zfill(10)}" + + +def xform_dbkey_to_report_id(audit_header, dbkey): + # month = audit_header.fyenddate.split('-')[1] + # 2022JUN0001000003 + # We start new audits at 1 million. + # So, we want 10 digits, and zero-pad for + # historic DBKEY report_ids + dt = xform_census_date_to_datetime(audit_header.FYENDDATE) + return f"{audit_header.AUDITYEAR}-{dt.month:02}-CENSUS-{dbkey.zfill(10)}" diff --git a/backend/census_historical_migration/sac_general_lib/sac_creator.py b/backend/census_historical_migration/sac_general_lib/sac_creator.py index 0d983bc827..8166c4332a 100644 --- a/backend/census_historical_migration/sac_general_lib/sac_creator.py +++ b/backend/census_historical_migration/sac_general_lib/sac_creator.py @@ -56,13 +56,13 @@ def _create_sac(user, dbkey): Access.objects.create( sac=sac, user=user, - email="fac-census-migration-auditee-official@auditee.org", # user.email, + email=user.email, role="certifying_auditee_contact", ) Access.objects.create( sac=sac, user=user, - email="fac-census-migration-auditor-official@auditor.org", # user.email, + email="fac-census-migration-auditor-official@fac.gsa.gov", role="certifying_auditor_contact", ) diff --git a/backend/census_historical_migration/sac_general_lib/utils.py b/backend/census_historical_migration/sac_general_lib/utils.py index 92f218b73a..299ed94f11 100644 --- a/backend/census_historical_migration/sac_general_lib/utils.py +++ b/backend/census_historical_migration/sac_general_lib/utils.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import date, datetime from census_historical_migration.transforms.xform_string_to_date import string_to_date from census_historical_migration.transforms.xform_string_to_string import ( string_to_string, @@ -54,3 +54,11 @@ def _census_date_to_datetime(cd): month = lookup[month_abbr] return date(int(year) + 2000, month, int(day)) + + +def xform_census_date_to_datetime(date_string): + """Convert a census date string from '%m/%d/%Y %H:%M:%S' format to 'YYYY-MM-DD' format.""" + # Parse the string into a datetime object + dt = datetime.strptime(date_string, "%m/%d/%Y %H:%M:%S") + # Extract and return the date part + return dt.date() diff --git a/backend/census_historical_migration/test_excel_creation.py b/backend/census_historical_migration/test_excel_creation.py index 8d7f9c8bf3..b6aaafe53f 100644 --- a/backend/census_historical_migration/test_excel_creation.py +++ b/backend/census_historical_migration/test_excel_creation.py @@ -1,11 +1,21 @@ +from django.conf import settings +from census_historical_migration.base_field_maps import ( + SheetFieldMap, + WorkbookFieldInDissem, +) from census_historical_migration.workbooklib.excel_creation_utils import ( + apply_conversion_function, + get_range_values, + get_ranges, set_range, ) - +from model_bakery import baker +from random import randint from django.test import TestCase from openpyxl import Workbook from openpyxl.utils import quote_sheetname, absolute_coordinate from openpyxl.workbook.defined_name import DefinedName +from census_historical_migration.models import ELECAUDITS as Audits class ExcelCreationTests(TestCase): @@ -143,3 +153,89 @@ def test_set_range_ws_missing(self): wb.defined_names.add(defn) self.assertRaises(KeyError, set_range, wb, self.range_name, ["bar"]) + + +class TestApplyConversionFunction(TestCase): + def test_string_conversion(self): + """Test that a string is returned unchanged""" + self.assertEqual(apply_conversion_function("test", "default", str), "test") + + def test_int_conversion(self): + """Test that an int is properly returned""" + self.assertEqual(apply_conversion_function("123", "default", int), 123) + + def test_custom_conversion(self): + """Test that a custom conversion function is properly applied""" + self.assertEqual( + apply_conversion_function("test", "default", lambda x: x.upper()), "TEST" + ) + + def test_default_value(self): + """Test that a default value is returned when the input is None""" + self.assertEqual(apply_conversion_function("", "default", str), "default") + + def test_none_with_no_default(self): + """Test that an empty string is returned when the input is None and no default is provided""" + self.assertEqual(apply_conversion_function(None, None, str), "") + + +class TestGetRanges(TestCase): + # Because the models used here are not related to the default database, + # we need to set 'databases' to include all database aliases. This ensures + # that the test case is aware of all the databases defined in the project's + # settings and can interact with them accordingly. + databases = {db_key for db_key in settings.DATABASES.keys()} + + def setUp(self): + """Set up mock mappings and values""" + self.mock_mappings = [ + SheetFieldMap( + "fake_range_name", "AUDITYEAR", WorkbookFieldInDissem, None, str + ), + SheetFieldMap( + "another_fake_range_name", "DBKEY", WorkbookFieldInDissem, None, str + ), + ] + + # Creating mock instances of the Audits model + self.mock_values = baker.make(Audits, _quantity=3) + self.random_year = randint(2016, 2022) # nosec + for audit in self.mock_values: + audit.AUDITYEAR = str(self.random_year) + audit.DBKEY = str(randint(20000, 21000)) # nosec + + def test_get_ranges(self): + """Test that the correct values are returned for each mapping""" + result = get_ranges(self.mock_mappings, self.mock_values) + + expected = [ + { + "name": "fake_range_name", + "values": [str(self.random_year) for _ in range(3)], + }, + { + "name": "another_fake_range_name", + "values": [audit.DBKEY for audit in self.mock_values], + }, + ] + + self.assertEqual(result, expected) + + def test_get_range_values(self): + """Test that get_range_values returns correct values for a given name""" + # First, get the ranges + ranges = get_ranges(self.mock_mappings, self.mock_values) + + # Test for a valid range name + fake_range_values = get_range_values(ranges, "fake_range_name") + self.assertEqual(fake_range_values, [str(self.random_year) for _ in range(3)]) + + # Test for another valid range name + another_range_values = get_range_values(ranges, "another_fake_range_name") + self.assertEqual( + another_range_values, [audit.DBKEY for audit in self.mock_values] + ) + + # Test for an invalid range name + invalid_range_values = get_range_values(ranges, "non_existent_range") + self.assertIsNone(invalid_range_values) diff --git a/backend/census_historical_migration/test_models.py b/backend/census_historical_migration/test_models.py new file mode 100644 index 0000000000..235ab11b7b --- /dev/null +++ b/backend/census_historical_migration/test_models.py @@ -0,0 +1,31 @@ +from django.test import TestCase +from django.conf import settings + +from model_bakery import baker + +from .models import ELECAUDITHEADER, ReportMigrationStatus, MigrationErrorDetail + + +class CensusHistoricalMigrationTests(TestCase): + databases = {k for k in settings.DATABASES.keys()} + + def test_can_load_elecauditheader_model(self): + gen = ELECAUDITHEADER.objects.all() + self.assertIsNotNone(gen) + baker.make(ELECAUDITHEADER).save() + gen = ELECAUDITHEADER.objects.all() + self.assertEquals(len(gen), 1) + + def test_can_load_report_migration_status_model(self): + report_migration_status = ReportMigrationStatus.objects.all() + self.assertIsNotNone(report_migration_status) + baker.make(ReportMigrationStatus).save() + report_migration_status = ReportMigrationStatus.objects.all() + self.assertEquals(len(report_migration_status), 1) + + def test_can_load_migration_error_detail_model(self): + migration_error_detail = MigrationErrorDetail.objects.all() + self.assertIsNotNone(migration_error_detail) + baker.make(MigrationErrorDetail).save() + migration_error_detail = MigrationErrorDetail.objects.all() + self.assertEquals(len(migration_error_detail), 1) diff --git a/backend/census_historical_migration/test_workbooks_should_pass.py b/backend/census_historical_migration/test_workbooks_should_pass.py new file mode 100644 index 0000000000..9da9874af5 --- /dev/null +++ b/backend/census_historical_migration/test_workbooks_should_pass.py @@ -0,0 +1,17 @@ +from django.test import SimpleTestCase +import os +from functools import reduce + +from audit.test_workbooks_should_pass import process_workbook_set + + +class PassingWorkbooks(SimpleTestCase): + def test_passing_workbooks(self): + workbook_sets = reduce( + os.path.join, + ["census_historical_migration", "fixtures", "workbooks", "should_pass"], + ) + for dirpath, dirnames, _ in os.walk(workbook_sets): + for workbook_set in dirnames: + print("Walking ", workbook_set) + process_workbook_set(os.path.join(dirpath, workbook_set)) diff --git a/backend/census_historical_migration/transforms/xform_string_to_string.py b/backend/census_historical_migration/transforms/xform_string_to_string.py index e6726e958b..f947097f3d 100644 --- a/backend/census_historical_migration/transforms/xform_string_to_string.py +++ b/backend/census_historical_migration/transforms/xform_string_to_string.py @@ -1,6 +1,13 @@ def string_to_string(value): - """Converts a string to a string.""" - if isinstance(value, str): - return value.strip() - else: + """ + Converts a string to a trimmed string. Returns an empty string if the input + is empty or 'nan'.""" + if value is None: + return "" + if not isinstance(value, str): raise ValueError(f"Expected string, got {type(value).__name__}") + trimmed_value = value.strip() + # FIXME-MSHD: When some CSV files are loaded + # to Postgres DB, empty string are being converted into 'nan' + # This is a temporary fix to handle the issue, we need to investigate this further. + return "" if trimmed_value in ["nan"] else trimmed_value diff --git a/backend/census_historical_migration/workbooklib/census_models/census.py b/backend/census_historical_migration/workbooklib/census_models/census.py index aef030d8a4..76cbd1e277 100644 --- a/backend/census_historical_migration/workbooklib/census_models/census.py +++ b/backend/census_historical_migration/workbooklib/census_models/census.py @@ -14,7 +14,7 @@ def model_module_path(model, year): def dynamic_import(mod, year): - name = model_module_path(mod, year) + name = model_module_path(mod, year[-2:]) components = name.split(".") mod = __import__(components[0]) for comp in components[1:]: diff --git a/backend/census_historical_migration/workbooklib/end_to_end_core.py b/backend/census_historical_migration/workbooklib/end_to_end_core.py index 68a418e0bc..a1413032c6 100644 --- a/backend/census_historical_migration/workbooklib/end_to_end_core.py +++ b/backend/census_historical_migration/workbooklib/end_to_end_core.py @@ -1,5 +1,4 @@ from census_historical_migration.exception_utils import DataMigrationError -from users.models import User import argparse import logging import sys @@ -8,8 +7,8 @@ import os import jwt import requests -from pprint import pprint from datetime import datetime +import traceback from census_historical_migration.workbooklib.workbook_builder_loader import ( workbook_builder_loader, @@ -174,14 +173,23 @@ def api_check(json_test_tables): # logger.info(f"{get_api_values(endpoint, report_id, f)}") api_values = get_api_values(endpoint, report_id, f) this_api_value = api_values[row_ndx] - this_field_value = row["values"][field_ndx] - eq = check_equality(this_field_value, this_api_value) - if not eq: + # Check if field_ndx exists in row["values"] + if field_ndx < len(row["values"]): + this_field_value = row["values"][field_ndx] + eq = check_equality(this_field_value, this_api_value) + if not eq: + logger.info( + f"Does not match. [eq {eq}] [field {f}] [field val {this_field_value}] != [api val {this_api_value}]" + ) + equality_results.append(eq) + else: + # Log a message if field_ndx does not exist logger.info( - f"Does not match. [eq {eq}] [field {f}] [field val {this_field_value}] != [api val {this_api_value}]" + f"Index {field_ndx} out of range for 'values' in row. Max index is {len(row['values']) - 1}" + ) + logger.info( + f"Field '{f}' with value '{this_api_value}' at index '{field_ndx}' is missing from test tables 'values'." ) - equality_results.append(eq) - if all(equality_results): count(summary, "correct_fields") else: @@ -192,41 +200,41 @@ def api_check(json_test_tables): return combined_summary -def generate_workbooks(user, dbkey, year): - entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format( - dbkey=dbkey, year=year, date=datetime.now() - ) - sac = setup_sac(user, entity_id, dbkey) - if sac.general_information["audit_type"] == "alternative-compliance-engagement": - print(f"Skipping ACE audit: {dbkey}") - raise DataMigrationError("Skipping ACE audit") - else: - builder_loader = workbook_builder_loader(user, sac, dbkey, year) - json_test_tables = [] - for section, fun in sections_to_handlers.items(): - # FIXME: Can we conditionally upload the addl' and secondary workbooks? - (_, json, _) = builder_loader(fun, section) - json_test_tables.append(json) - _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf") - step_through_certifications(sac) - - # shaped_sac = sac_validation_shape(sac) - # result = submission_progress_check(shaped_sac, sar=None, crossval=False) - # print(result) - - errors = sac.validate_cross() - pprint(errors.get("errors", "No errors found in cross validation")) - - disseminate(sac, year) - # pprint(json_test_tables) - combined_summary = api_check(json_test_tables) - logger.info(combined_summary) - - -def run_end_to_end(email, dbkey, year): +def run_end_to_end(user, dbkey, year, result): try: - user = User.objects.get(email=email) - except User.DoesNotExist: - logger.info("No user found for %s, have you logged in once?", email) - return - generate_workbooks(user, dbkey, year) + entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format( + dbkey=dbkey, year=year, date=datetime.now() + ) + sac = setup_sac(user, entity_id, dbkey) + + if sac.general_information["audit_type"] == "alternative-compliance-engagement": + print(f"Skipping ACE audit: {dbkey}") + raise DataMigrationError("Skipping ACE audit") + else: + builder_loader = workbook_builder_loader(user, sac, dbkey, year) + json_test_tables = [] + + for section, fun in sections_to_handlers.items(): + # FIXME: Can we conditionally upload the addl' and secondary workbooks? + (_, json, _) = builder_loader(fun, section) + json_test_tables.append(json) + + _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf") + step_through_certifications(sac) + + errors = sac.validate_cross() + if errors.get("errors"): + result["errors"].append(f"{errors.get('errors')}") + return + + disseminate(sac, year) + combined_summary = api_check(json_test_tables) + logger.info(combined_summary) + + result["success"].append(f"{sac.report_id} created") + except Exception as exc: + tb = traceback.extract_tb(sys.exc_info()[2]) + for frame in tb: + print(f"{frame.filename}:{frame.lineno} {frame.name}: {frame.line}") + + result["errors"].append(f"{exc}") diff --git a/backend/census_historical_migration/workbooklib/excel_creation_utils.py b/backend/census_historical_migration/workbooklib/excel_creation_utils.py index c5a14c9023..16f8620c19 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation_utils.py +++ b/backend/census_historical_migration/workbooklib/excel_creation_utils.py @@ -1,11 +1,19 @@ +from census_historical_migration.transforms.xform_string_to_string import ( + string_to_string, +) +from census_historical_migration.transforms.xform_string_to_int import string_to_int from census_historical_migration.exception_utils import DataMigrationError from census_historical_migration.base_field_maps import WorkbookFieldInDissem from census_historical_migration.workbooklib.templates import sections_to_template_paths from census_historical_migration.sac_general_lib.report_id_generator import ( dbkey_to_report_id, + xform_dbkey_to_report_id, +) +from census_historical_migration.models import ( + ELECAUDITS as Audits, + ELECAUDITHEADER as AuditHeader, ) -from playhouse.shortcuts import model_to_dict from openpyxl.utils.cell import ( rows_from_range, coordinate_from_string, @@ -61,11 +69,63 @@ def set_range(wb, range_name, values, default=None, conversion_fun=str): col_str, row = coordinate_from_string(cell) # ('B12',) -> 'B', 12 col = column_index_from_string(col_str) # 'B' -> 2 + # Check for the type and apply the correct conversion method + converted_value = apply_conversion_function(value, default, conversion_fun) # Set the value of the cell - converted_value = conversion_fun(value) if value else default or "" ws.cell(row=row, column=col, value=converted_value) +def apply_conversion_function(value, default, conversion_function): + """ + Helper to apply a conversion function to a value, or use a default value + """ + if value: + if conversion_function is str: + new_value = string_to_string(value) + elif conversion_function is int: + new_value = string_to_int(value) + else: + new_value = conversion_function(value) + else: + new_value = default or "" + return new_value + + +def get_range_values(ranges, name): + """ + Helper to get the values linked to a particular range, identified by its name.""" + for item in ranges: + if item["name"] == name: + return item["values"] + return None + + +def get_ranges(mappings, values): + """ + Helper to get range of values.The method iterates over a collection of mappings, applying a conversion + function to constructs a list of dictionaries, each containing a name and a list of + transformed values.""" + ranges = [] + for mapping in mappings: + ranges.append( + { + "name": mapping.in_sheet, + "values": list( + map( + lambda v: apply_conversion_function( + getattr(v, mapping.in_db), + mapping.default, + mapping.type, + ), + values, + ) + ), + } + ) + return ranges + + +# FIXME-MSHD: Remove this function once we switch all workbook generators to using census models def set_uei(Gen, wb, dbkey): g = Gen.select().where(Gen.dbkey == dbkey).get() if g.uei: @@ -75,6 +135,22 @@ def set_uei(Gen, wb, dbkey): return g +def set_workbook_uei(workbook, uei): + """Sets the UEI value in the workbook's designated UEI cell""" + if not uei: + raise DataMigrationError("UEI value is missing or invalid.") + set_range(workbook, "auditee_uei", [uei]) + + +def get_audit_header(dbkey): + """Returns the AuditHeader instance for the given dbkey.""" + try: + audit_header = AuditHeader.objects.get(DBKEY=dbkey) + except AuditHeader.DoesNotExist: + raise DataMigrationError(f"No audit header record found for dbkey: {dbkey}") + return audit_header + + def map_simple_columns(wb, mappings, values): len_passed_in = len(mappings) unique_fields = set() @@ -94,7 +170,7 @@ def map_simple_columns(wb, mappings, values): set_range( wb, m.in_sheet, - map(lambda v: model_to_dict(v)[m.in_db], values), + map(lambda v: getattr(v, m.in_db), values), m.default, m.type, ) @@ -125,27 +201,36 @@ def get_template_name_for_section(section): def generate_dissemination_test_table(Gen, api_endpoint, dbkey, mappings, objects): table = {"rows": list(), "singletons": dict()} table["endpoint"] = api_endpoint - table["report_id"] = dbkey_to_report_id(Gen, dbkey) + table["report_id"] = ( + dbkey_to_report_id(Gen, dbkey) + if not isinstance( + Gen, AuditHeader + ) # FIXME-MSHD: This hack is necessary until we switch all workbook generators to using census models. We may want to get rid of generate_dissemination_test_table at some point (see comment in generate_federal_awards) + else xform_dbkey_to_report_id(Gen, dbkey) + ) for o in objects: - as_dict = model_to_dict(o) test_obj = {} test_obj["fields"] = [] test_obj["values"] = [] for m in mappings: # What if we only test non-null values? - if ((m.in_db in as_dict) and as_dict[m.in_db] is not None) and ( - as_dict[m.in_db] != "" - ): + raw_value = getattr(o, m.in_db, None) + attribute_value = apply_conversion_function(raw_value, m.default, m.type) + if (attribute_value is not None) and (attribute_value != ""): if m.in_dissem == WorkbookFieldInDissem: - # print(f'in_sheet {m.in_sheet} <- {as_dict[m.in_db]}') + # print(f'in_sheet {m.in_sheet} <- {attribute_value}') test_obj["fields"].append(m.in_sheet) # The typing must be applied here as well, as in the case of # type_requirement, it alphabetizes the value... - test_obj["values"].append(m.type(as_dict[m.in_db])) + test_obj["values"].append(m.type(attribute_value)) else: - # print(f'in_dissem {m.in_dissem} <- {as_dict[m.in_db]}') + # print(f'in_dissem {m.in_dissem} <- {attribute_value}') test_obj["fields"].append(m.in_dissem) - test_obj["values"].append(m.type(as_dict[m.in_db])) + test_obj["values"].append(m.type(attribute_value)) table["rows"].append(test_obj) return table + + +def get_audits(dbkey): + return Audits.objects.filter(DBKEY=dbkey).order_by("ID") diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py index 8fc239bf39..8ddae318d7 100644 --- a/backend/census_historical_migration/workbooklib/federal_awards.py +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -1,5 +1,12 @@ +from census_historical_migration.transforms.xform_string_to_string import ( + string_to_string, +) from census_historical_migration.workbooklib.excel_creation_utils import ( - set_uei, + get_audit_header, + get_audits, + get_range_values, + get_ranges, + set_workbook_uei, map_simple_columns, generate_dissemination_test_table, set_range, @@ -9,13 +16,15 @@ WorkbookFieldInDissem, ) from census_historical_migration.workbooklib.templates import sections_to_template_paths -from census_historical_migration.workbooklib.census_models.census import dynamic_import from audit.fixtures.excel import FORM_SECTIONS from config import settings - +from census_historical_migration.models import ( + ELECAUDITS as Audits, + ELECPASSTHROUGH as Passthrough, +) +from django.db.models import Q import openpyxl as pyxl import json -import re import logging @@ -31,209 +40,248 @@ def if_zero_empty(v): mappings = [ SheetFieldMap( - "program_name", "federalprogramname", "federal_program_name", None, str + "federal_agency_prefix", "CFDA_PREFIX", WorkbookFieldInDissem, None, str + ), + SheetFieldMap( + "three_digit_extension", "CFDA_EXT", "federal_award_extension", None, str + ), + SheetFieldMap( + "program_name", "FEDERALPROGRAMNAME", "federal_program_name", None, str + ), + SheetFieldMap( + "state_cluster_name", "STATECLUSTERNAME", WorkbookFieldInDissem, None, str ), SheetFieldMap( - "state_cluster_name", "stateclustername", WorkbookFieldInDissem, None, str + "federal_program_total", "PROGRAMTOTAL", WorkbookFieldInDissem, 0, int ), SheetFieldMap( - "federal_program_total", "programtotal", WorkbookFieldInDissem, 0, int + "additional_award_identification", + "AWARDIDENTIFICATION", + WorkbookFieldInDissem, + None, + str, ), - SheetFieldMap("cluster_total", "clustertotal", WorkbookFieldInDissem, 0, int), - SheetFieldMap("is_guaranteed", "loans", "is_loan", None, str), + SheetFieldMap("cluster_total", "CLUSTERTOTAL", WorkbookFieldInDissem, 0, int), + SheetFieldMap("is_guaranteed", "LOANS", "is_loan", None, str), + # In the intake process, we initially use convert_to_stripped_string to convert IR values into strings, + # and then apply specific functions like convert_loan_balance_to_integers_or_na to convert particular fields + # such as loan_balance_at_audit_period_end into their appropriate formats. Therefore, it's suitable to process + # this column as a string here because treating it as an integer would be incorrect due to the presence of 'N/A' values. + # Any values like 'n/a', if present, may initially fail to process but will be addressed through data transformation + # in subsequent iterations of the data migration process. SheetFieldMap( - "loan_balance_at_audit_period_end", "loanbalance", "loan_balance", None, int + "loan_balance_at_audit_period_end", "LOANBALANCE", "loan_balance", None, str ), - SheetFieldMap("is_direct", "direct", WorkbookFieldInDissem, None, str), - SheetFieldMap("is_passed", "passthroughaward", "is_passthrough_award", None, str), + SheetFieldMap("is_direct", "DIRECT", WorkbookFieldInDissem, None, str), + SheetFieldMap("is_passed", "PASSTHROUGHAWARD", "is_passthrough_award", None, str), SheetFieldMap( "subrecipient_amount", - "passthroughamount", + "PASSTHROUGHAMOUNT", "passthrough_amount", None, if_zero_empty, ), - SheetFieldMap("is_major", "majorprogram", WorkbookFieldInDissem, None, str), - SheetFieldMap("audit_report_type", "typereport_mp", "audit_report_type", None, str), + SheetFieldMap("is_major", "MAJORPROGRAM", WorkbookFieldInDissem, None, str), + SheetFieldMap("audit_report_type", "TYPEREPORT_MP", "audit_report_type", None, str), SheetFieldMap( - "number_of_audit_findings", "findingscount", "findings_count", 0, int + "number_of_audit_findings", "FINDINGSCOUNT", "findings_count", 0, int ), - SheetFieldMap("amount_expended", "amount", WorkbookFieldInDissem, 0, int), + SheetFieldMap("amount_expended", "AMOUNT", WorkbookFieldInDissem, 0, int), ] -def get_list_index(all, index): +def get_list_index(all_items, index): counter = 0 - for o in list(all): - if o.index == index: + for item in list(all_items): + if item.ID == index: return counter else: counter += 1 return -1 -def int_or_na(o): - if o == "N/A": - return o - elif isinstance(o, int): - return int(o) - else: - return "N/A" - - -def _generate_cluster_names(Cfda, cfdas, valid_json): +def _generate_cluster_names( + audits: list[Audits], +) -> tuple[list[str], list[str], list[str]]: + """Reconstructs the cluster names for each audit in the provided list.""" + # Patch the clusternames. They used to be allowed to enter anything + # they wanted. + valid_file = open(f"{settings.BASE_DIR}/schemas/source/base/ClusterNames.json") + valid_json = json.load(valid_file) cluster_names = [] state_cluster_names = [] other_cluster_names = [] - cfda: Cfda - for cfda in cfdas: - if cfda.clustername is None: + for audit in audits: + cluster_name = string_to_string(audit.CLUSTERNAME) + state_cluster_name = string_to_string(audit.STATECLUSTERNAME) + other_cluster_name = string_to_string(audit.OTHERCLUSTERNAME) + if not cluster_name: cluster_names.append("N/A") other_cluster_names.append("") state_cluster_names.append("") - elif cfda.clustername == "STATE CLUSTER": - cluster_names.append(cfda.clustername) - state_cluster_names.append(cfda.stateclustername) + elif cluster_name == "STATE CLUSTER": + cluster_names.append(cluster_name) + state_cluster_names.append(state_cluster_name) other_cluster_names.append("") - elif cfda.clustername == "OTHER CLUSTER NOT LISTED ABOVE": - cluster_names.append(cfda.clustername) - other_cluster_names.append(cfda.otherclustername) + elif cluster_name == "OTHER CLUSTER NOT LISTED ABOVE": + cluster_names.append(cluster_name) + other_cluster_names.append(other_cluster_name) state_cluster_names.append("") - elif cfda.clustername in valid_json["cluster_names"]: - cluster_names.append(cfda.clustername) + elif cluster_name in valid_json["cluster_names"]: + cluster_names.append(cluster_name) other_cluster_names.append("") state_cluster_names.append("") else: - logger.debug(f"Cluster {cfda.clustername} not in the list. Replacing.") + logger.debug(f"Cluster {cluster_name} not in the list. Replacing.") cluster_names.append("OTHER CLUSTER NOT LISTED ABOVE") - other_cluster_names.append(f"{cfda.clustername}") + other_cluster_names.append(f"{cluster_name}") state_cluster_names.append("") return (cluster_names, other_cluster_names, state_cluster_names) -def _fix_addl_award_identification(Cfda, cfdas, dbkey): - addls = ["" for x in list(range(0, len(cfdas)))] - for cfda in ( - Cfda.select() - .where( - (Cfda.dbkey == dbkey) - & ( - (Cfda.cfda % "%U%") - | (Cfda.cfda % "%u%") - | (Cfda.cfda % "%rd%") - | (Cfda.cfda % "%RD%") - ) - ) - .order_by(Cfda.index) +def _get_full_cfdas(audits): + """ + This function constructs the full CFDA numbers by concatenating the CFDA_PREFIX + and CFDA_EXT attributes of each audit object, separated by a dot. + """ + # audit.CFDA is not used here because it does not always match f"{audit.CFDA_PREFIX}.{audit.CFDA_EXT}" + return [f"{audit.CFDA_PREFIX}.{audit.CFDA_EXT}" for audit in audits] + + +# The functionality of _fix_passthroughs has been split into two separate functions: +# _get_passthroughs and _xform_populate_default_passthrough_values. Currently, _get_passthroughs is being +# used in place of _fix_passthroughs. +def _get_passthroughs(audits): + """ + Retrieves the passthrough names and IDs for a given list of audits. + For each audit in the provided list, this function queries the Passthrough model to find + records matching the DBKEY and ELECAUDITSID of the audit. It then compiles lists of + passthrough names and IDs, joined by a pipe '|' if multiple are found. + """ + passthrough_names = [""] * len(audits) + passthrough_ids = [""] * len(audits) + + for index, audit in enumerate(audits): + passthroughs = Passthrough.objects.filter( + DBKEY=audit.DBKEY, ELECAUDITSID=audit.ELECAUDITSID + ).order_by("ID") + # This may look like data transformation but it is not exactly the case. + # In the audit worksheet, users can enter multiple names (or IDs) separated by a pipe '|' in a single cell. + # We are simply reconstructing this pipe separated data here. + names = [] + ids = [] + for passthrough in passthroughs: + passthrough_name = string_to_string(passthrough.PASSTHROUGHNAME) + passthrough_id = string_to_string(passthrough.PASSTHROUGHID) + if passthrough_name: + names.append(passthrough_name) + if passthrough_id: + ids.append(passthrough_id) + + passthrough_names[index] = "|".join(names) if names else "" + passthrough_ids[index] = "|".join(ids) if ids else "" + return (passthrough_names, passthrough_ids) + + +# FIXME - MSHD: _xform_populate_default_passthrough_values is currently unused as unrequired data transformation +# will not be part of the first iteration of the data migration process. +# Only required (MUST) data transformation will be part of the first iteration. +def _xform_populate_default_passthrough_values( + passthrough_names, passthrough_ids, audits +): + """ + Automatically fills in default values for empty passthrough names and IDs. + Iterates over a list of audits and their corresponding passthrough names and IDs. + If the audit's DIRECT attribute is "N" and the passthrough name or ID is empty, + it fills in a default value indicating that no passthrough name or ID was provided. + """ + for index, audit, name, id in zip( + range(len(audits)), audits, passthrough_names, passthrough_ids ): - if cfda.awardidentification is None or len(cfda.awardidentification) < 1: - addls[ - get_list_index(cfdas, cfda.index) - ] = f"ADDITIONAL AWARD INFO - DBKEY {dbkey}" - else: - addls[get_list_index(cfdas, cfda.index)] = cfda.awardidentification - return addls - - -def _fix_pfixes(cfdas): - # Map things with transformations - prefixes = map(lambda v: (v.cfda).split(".")[0], cfdas) - # prefixes = map(lambda v: f'0{v}' if int(v) < 10 else v, prefixes) - # Truncate any nastiness in the CFDA extensions to three characters. - extensions = map(lambda v: ((v.cfda).split(".")[1])[:3].upper(), cfdas) - extensions = map( - lambda v: v - if re.search("^(RD|RD[0-9]|[0-9]{3}[A-Za-z]{0,1}|U[0-9]{2})$", v) - else "000", - extensions, - ) - return (prefixes, extensions, map(lambda v: v.cfda, cfdas)) - - -def _fix_passthroughs(Cfda, Passthrough, cfdas, dbkey): - passthrough_names = ["" for x in list(range(0, len(cfdas)))] - passthrough_ids = ["" for x in list(range(0, len(cfdas)))] - ls = Cfda.select().where(Cfda.dbkey == dbkey).order_by(Cfda.index) - cfda: Cfda - for cfda in ls: - pnq = Passthrough() - if cfda.direct == "Y": - pnq.passthroughname = "" - pnq.passthroughid = "" - if cfda.direct == "N": - try: - pnq = ( - Passthrough.select().where( - (Passthrough.dbkey == cfda.dbkey) - & (Passthrough.elecauditsid == cfda.elecauditsid) - ) - ).get() - except Exception as e: - print(e) - pnq.passthroughname = "EXCEPTIONAL PASSTHROUGH NAME" - pnq.passthroughid = "EXCEPTIONAL PASSTHROUGH ID" - - name = pnq.passthroughname - if name is None: - name = "" - name = name.rstrip() - if name == "" and cfda.direct == "N": - passthrough_names[ - get_list_index(cfdas, cfda.index) - ] = "NO PASSTHROUGH NAME PROVIDED" + if audit.DIRECT == "N" and name == "": + passthrough_names[index] = "NO PASSTHROUGH NAME PROVIDED" + if audit.DIRECT == "N" and id == "": + passthrough_ids[index] = "NO PASSTHROUGH ID PROVIDED" + return (passthrough_names, passthrough_ids) + + +# FIXME - MSHD: _xform_populate_default_loan_balance is currently unused +# as unrequired data transformation will not be part of the first iteration +# of the data migration process. +def _xform_populate_default_loan_balance(loans_at_end, audits): + """ + Automatically fills in default values for empty loan balances. + Iterates over a list of audits and their corresponding loan balances. + If the audit's LOANS attribute is "Y" and the loan balance is empty, + it fills in a default value indicating that no loan balance was provided.""" + for ndx, audit in zip(range(len(audits)), audits, loans_at_end): + if audit.LOANS == "Y": + if audit.LOANBALANCE is None: + loans_at_end[ + ndx + ] = 1 # FIXME - MSHD: This value requires team approval. + # There are cases (dbkeys 148665/150450) with balance = -1.0, how do we handle this? else: - passthrough_names[get_list_index(cfdas, cfda.index)] = name - - id = pnq.passthroughid - if id is None: - id = "" - id = id.rstrip() - if id == "" and cfda.direct == "N": - passthrough_ids[ - get_list_index(cfdas, cfda.index) - ] = "NO PASSTHROUGH ID PROVIDED" + if audit.LOANBALANCE is not None: + loans_at_end[ndx] = "" + return loans_at_end + + +# FIXME - MSHD: _xform_populate_default_award_identification_values is currently unused +# as unrequired data transformation will not be part of the first iteration +# of the data migration process. +def _xform_populate_default_award_identification_values(audits, dbkey): + """ + Automatically fills in default values for empty additional award identifications. + Iterates over a list of audits and their corresponding additional award identifications. + If the audit's CFDA attribute contains "U" or "u" or "rd" or "RD" and the award identification is empty, + it fills in a default value indicating that no award identification was provided. + """ + addl_award_identifications = [""] * len(audits) + filtered_audits = Audits.objects.filter( + Q(DBKEY=dbkey) & (Q(CFDA__icontains="U") | Q(CFDA__icontains="rd")) + ).order_by("ID") + for audit in filtered_audits: + if audit.AWARDIDENTIFICATION is None or len(audit.AWARDIDENTIFICATION) < 1: + addl_award_identifications[ + get_list_index(audits, audit.ID) + ] = f"ADDITIONAL AWARD INFO - DBKEY {dbkey}" else: - passthrough_ids[get_list_index(cfdas, cfda.index)] = pnq.passthroughid - - return (passthrough_names, passthrough_ids) + addl_award_identifications[ + get_list_index(audits, audit.ID) + ] = audit.AWARDIDENTIFICATION + return addl_award_identifications def generate_federal_awards(dbkey, year, outfile): + """ + Generates a federal awards workbook for all awards associated with a given dbkey. + + Note: This function assumes that all the audit information in the database + is for the same year. + """ logger.info(f"--- generate federal awards {dbkey} {year} ---") - Gen = dynamic_import("Gen", year) - Passthrough = dynamic_import("Passthrough", year) - Cfda = dynamic_import("Cfda", year) + wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED] ) - # In sheet : in DB - g = set_uei(Gen, wb, dbkey) + audit_header = get_audit_header(dbkey) - cfdas = Cfda.select().where(Cfda.dbkey == dbkey).order_by(Cfda.index) - map_simple_columns(wb, mappings, cfdas) + set_workbook_uei(wb, audit_header.UEI) - # Patch the clusternames. They used to be allowed to enter anything - # they wanted. - valid_file = open(f"{settings.BASE_DIR}/schemas/source/base/ClusterNames.json") - valid_json = json.load(valid_file) + audits = get_audits(dbkey) + + map_simple_columns(wb, mappings, audits) (cluster_names, other_cluster_names, state_cluster_names) = _generate_cluster_names( - Cfda, cfdas, valid_json + audits ) set_range(wb, "cluster_name", cluster_names) set_range(wb, "other_cluster_name", other_cluster_names) - # Fix the additional award identification. If they had a "U", we want - # to see something in the addl. column. - addls = _fix_addl_award_identification(Cfda, cfdas, dbkey) - set_range(wb, "additional_award_identification", addls) - - (prefixes, extensions, full_cfdas) = _fix_pfixes(cfdas) - set_range(wb, "federal_agency_prefix", prefixes) - set_range(wb, "three_digit_extension", extensions) - # We need a `cfda_key` as a magic column for the summation logic to work/be checked. + full_cfdas = _get_full_cfdas(audits) set_range(wb, "cfda_key", full_cfdas, conversion_fun=str) # We need `uniform_state_cluster_name` and `uniform_other_cluster_name` magic columns for cluster summation logic to work/be checked. @@ -250,9 +298,8 @@ def generate_federal_awards(dbkey, year, outfile): conversion_fun=str, ) - (passthrough_names, passthrough_ids) = _fix_passthroughs( - Cfda, Passthrough, cfdas, dbkey - ) + (passthrough_names, passthrough_ids) = _get_passthroughs(audits) + set_range(wb, "passthrough_name", passthrough_names) set_range(wb, "passthrough_identifying_number", passthrough_ids) @@ -260,56 +307,73 @@ def generate_federal_awards(dbkey, year, outfile): set_range( wb, "award_reference", - [f"AWARD-{n+1:04}" for n in range(len(passthrough_names))], + [f"AWARD-{n+1:04}" for n in range(len(audits))], ) # Total amount expended must be calculated and inserted total = 0 - for cfda in cfdas: - total += int(cfda.amount) - set_range(wb, "total_amount_expended", [total]) - - loansatend = list() - for ndx, cfda in enumerate( - Cfda.select().where((Cfda.dbkey == dbkey)).order_by(Cfda.index) - ): - if cfda.loans == "Y": - if cfda.loanbalance is None: - # loansatend.append("N/A") - loansatend.append(1) - else: - loansatend.append(cfda.loanbalance) - else: - loansatend.append("") - # set_range(wb, "loan_balance_at_audit_period_end", loansatend, type=int_or_na) - set_range(wb, "loan_balance_at_audit_period_end", loansatend, conversion_fun=int) + for audit in audits: + total += int(audit.AMOUNT) + set_range(wb, "total_amount_expended", [str(total)]) wb.save(outfile) + # FIXME - MSHD: The test table and the logic around it do not seem necessary to me. + # If there is any chance that the dissemination process allows bogus data to be disseminated, + # we should fix the dissemination process instead by reinforcing the validation logic (intake validation and cross-validation). + # I will create a ticket for the removal of this logic unless someone comes up with a strong reason to keep it. table = generate_dissemination_test_table( - Gen, "federal_awards", dbkey, mappings, cfdas + audit_header, "federal_awards", dbkey, mappings, audits ) award_counter = 1 + filtered_mappings = [ + mapping + for mapping in mappings + if mapping.in_sheet + in [ + "additional_award_identification", + "federal_agency_prefix", + "three_digit_extension", + ] + ] + ranges = get_ranges(filtered_mappings, audits) + prefixes = get_range_values(ranges, "federal_agency_prefix") + extensions = get_range_values(ranges, "three_digit_extension") + additional_award_identifications = get_range_values( + ranges, "additional_award_identification" + ) # prefix - for obj, pfix, ext, addl, cn, ocn in zip( - table["rows"], prefixes, extensions, addls, cluster_names, other_cluster_names + for ( + award, + prefix, + extension, + additional_identification, + cluster_name, + other_cluster_name, + ) in zip( + table["rows"], + prefixes, + extensions, + additional_award_identifications, + cluster_names, + other_cluster_names, ): - obj["fields"].append("federal_agency_prefix") - obj["values"].append(pfix) - obj["fields"].append("three_digit_extension") - obj["values"].append(ext) + award["fields"].append("federal_agency_prefix") + award["values"].append(prefix) + award["fields"].append("federal_award_extension") + award["values"].append(extension) # Sneak in the award number here - obj["fields"].append("award_reference") - obj["values"].append(f"AWARD-{award_counter:04}") - obj["fields"].append("additional_award_identification") - obj["values"].append(addl) - obj["fields"].append("cluster_name") - obj["values"].append(cn) - obj["fields"].append("other_cluster_name") - obj["fields"].append(ocn) + award["fields"].append("award_reference") + award["values"].append(f"AWARD-{award_counter:04}") + award["fields"].append("additional_award_identification") + award["values"].append(additional_identification) + award["fields"].append("cluster_name") + award["values"].append(cluster_name) + award["fields"].append("other_cluster_name") + award["fields"].append(other_cluster_name) award_counter += 1 - table["singletons"]["auditee_uei"] = g.uei + table["singletons"]["auditee_uei"] = audit_header.UEI table["singletons"]["total_amount_expended"] = total return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py index 1b133c53f5..0347f52af4 100644 --- a/backend/census_historical_migration/workbooklib/secondary_auditors.py +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -1,12 +1,13 @@ from census_historical_migration.workbooklib.excel_creation_utils import ( add_hyphen_to_zip, - set_uei, + get_audit_header, map_simple_columns, generate_dissemination_test_table, + set_workbook_uei, ) from census_historical_migration.base_field_maps import SheetFieldMap from census_historical_migration.workbooklib.templates import sections_to_template_paths -from census_historical_migration.workbooklib.census_models.census import dynamic_import +from census_historical_migration.models import ELECCPAS as Caps from audit.fixtures.excel import FORM_SECTIONS import openpyxl as pyxl @@ -17,39 +18,39 @@ mappings = [ SheetFieldMap( - "secondary_auditor_address_city", "cpacity", "address_city", None, str + "secondary_auditor_address_city", "CPACITY", "address_city", None, str ), SheetFieldMap( - "secondary_auditor_contact_name", "cpacontact", "contact_name", None, str + "secondary_auditor_contact_name", "CPACONTACT", "contact_name", None, str ), - SheetFieldMap("secondary_auditor_ein", "cpaein", "auditor_ein", None, str), + SheetFieldMap("secondary_auditor_ein", "CPAEIN", "auditor_ein", None, str), SheetFieldMap( - "secondary_auditor_contact_email", "cpaemail", "contact_email", None, str + "secondary_auditor_contact_email", "CPAEMAIL", "contact_email", None, str ), - SheetFieldMap("secondary_auditor_name", "cpafirmname", "auditor_name", None, str), + SheetFieldMap("secondary_auditor_name", "CPAFIRMNAME", "auditor_name", None, str), SheetFieldMap( - "secondary_auditor_contact_phone", "cpaphone", "contact_phone", None, str + "secondary_auditor_contact_phone", "CPAPHONE", "contact_phone", None, str ), SheetFieldMap( - "secondary_auditor_address_state", "cpastate", "address_state", None, str + "secondary_auditor_address_state", "CPASTATE", "address_state", None, str ), SheetFieldMap( "secondary_auditor_address_street", - "cpastreet1", + "CPASTREET1", "address_street", None, str, ), SheetFieldMap( "secondary_auditor_contact_title", - "cpatitle", + "CPATITLE", "contact_title", None, str, ), SheetFieldMap( "secondary_auditor_address_zipcode", - "cpazipcode", + "CPAZIPCODE", "address_zipcode", None, add_hyphen_to_zip, @@ -57,26 +58,35 @@ ] +def _get_secondary_auditors(dbkey): + return Caps.objects.filter(DBKEY=dbkey) + + def generate_secondary_auditors(dbkey, year, outfile): + """ + Generates secondary auditor workbook for a given dbkey. + """ logger.info(f"--- generate secondary auditors {dbkey} {year} ---") - Gen = dynamic_import("Gen", year) - Cpas = dynamic_import("Cpas", year) + wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.SECONDARY_AUDITORS] ) - g = set_uei(Gen, wb, dbkey) + audit_header = get_audit_header(dbkey) + + set_workbook_uei(wb, audit_header.UEI) - sec_cpas = Cpas.select().where(Cpas.dbkey == g.dbkey) + secondary_auditors = _get_secondary_auditors(dbkey) - map_simple_columns(wb, mappings, sec_cpas) + map_simple_columns(wb, mappings, secondary_auditors) wb.save(outfile) + # FIXME - MSHD: The logic below will most likely be removed, see comment in federal_awards.py table = generate_dissemination_test_table( - Gen, "secondary_auditor", dbkey, mappings, sec_cpas + audit_header, "secondary_auditor", dbkey, mappings, secondary_auditors ) - table["singletons"]["auditee_uei"] = g.uei + table["singletons"]["auditee_uei"] = audit_header.UEI return (wb, table) diff --git a/backend/dissemination/forms.py b/backend/dissemination/forms.py index 5db043b3ea..9b51cb52c8 100644 --- a/backend/dissemination/forms.py +++ b/backend/dissemination/forms.py @@ -15,6 +15,7 @@ class SearchForm(forms.Form): cog_or_oversight = forms.CharField(required=False) agency_name = forms.CharField(required=False) audit_year = forms.MultipleChoiceField(choices=AY_choices, required=False) + auditee_state = forms.CharField(required=False) # Display params limit = forms.CharField(required=False) diff --git a/backend/dissemination/migrations/0006_migrationchangerecord.py b/backend/dissemination/migrations/0006_migrationchangerecord.py new file mode 100644 index 0000000000..f013398668 --- /dev/null +++ b/backend/dissemination/migrations/0006_migrationchangerecord.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.6 on 2023-11-27 19:38 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + dependencies = [ + ("dissemination", "0005_alter_general_cognizant_agency_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="MigrationChangeRecord", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("audit_year", models.TextField(blank=True, null=True)), + ("dbkey", models.TextField(blank=True, null=True)), + ("report_id", models.TextField(blank=True, null=True)), + ( + "run_datetime", + models.DateTimeField(default=django.utils.timezone.now), + ), + ("census_data", models.JSONField(blank=True, null=True)), + ("gsa_fac_data", models.JSONField(blank=True, null=True)), + ("transformation_function", models.TextField(blank=True, null=True)), + ], + ), + ] diff --git a/backend/dissemination/models.py b/backend/dissemination/models.py index 50d933fcf9..b11d7ac840 100644 --- a/backend/dissemination/models.py +++ b/backend/dissemination/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.utils import timezone from . import docs @@ -505,3 +506,13 @@ class SecondaryAuditor(models.Model): report_id = models.TextField( REPORT_ID_FK_HELP_TEXT, ) + + +class MigrationChangeRecord(models.Model): + audit_year = models.TextField(blank=True, null=True) + dbkey = models.TextField(blank=True, null=True) + report_id = models.TextField(blank=True, null=True) + run_datetime = models.DateTimeField(default=timezone.now) + census_data = models.JSONField(blank=True, null=True) + gsa_fac_data = models.JSONField(blank=True, null=True) + transformation_function = models.TextField(blank=True, null=True) diff --git a/backend/dissemination/search.py b/backend/dissemination/search.py index 66fc7f215d..d9ffec1fe7 100644 --- a/backend/dissemination/search.py +++ b/backend/dissemination/search.py @@ -12,6 +12,7 @@ def search_general( cog_or_oversight=None, agency_name=None, audit_years=None, + auditee_state=None, include_private=False, ): query = Q() @@ -23,6 +24,7 @@ def search_general( query.add(_get_end_date_match_query(end_date), Q.AND) query.add(_get_cog_or_oversight_match_query(agency_name, cog_or_oversight), Q.AND) query.add(_get_audit_years_match_query(audit_years), Q.AND) + query.add(_get_auditee_state_match_query(auditee_state), Q.AND) if not include_private: query.add(Q(is_public=True), Q.AND) @@ -167,3 +169,10 @@ def _get_audit_years_match_query(audit_years): return Q() return Q(audit_year__in=audit_years) + + +def _get_auditee_state_match_query(auditee_state): + if not auditee_state: + return Q() + + return Q(auditee_state__in=[auditee_state]) diff --git a/backend/dissemination/templates/search.html b/backend/dissemination/templates/search.html index bba71d81e3..33aa9b568e 100644 --- a/backend/dissemination/templates/search.html +++ b/backend/dissemination/templates/search.html @@ -116,6 +116,19 @@

Filters

name="agency_name" value="{{ form.cleaned_data.agency_name }}" /> +
+ + +
{% comment %} Submission {% endcomment %}
diff --git a/backend/dissemination/test_models.py b/backend/dissemination/test_models.py new file mode 100644 index 0000000000..c010fb7687 --- /dev/null +++ b/backend/dissemination/test_models.py @@ -0,0 +1,14 @@ +from django.test import TestCase + +from model_bakery import baker + +from .models import MigrationChangeRecord + + +class MigrationChangeRecordTests(TestCase): + def test_can_load_migration_change_record_model(self): + migration_change_record = MigrationChangeRecord.objects.all() + self.assertIsNotNone(migration_change_record) + baker.make(MigrationChangeRecord).save() + migration_change_record = MigrationChangeRecord.objects.all() + self.assertEquals(len(migration_change_record), 1) diff --git a/backend/dissemination/test_search.py b/backend/dissemination/test_search.py index ddc3788faf..4be697f078 100644 --- a/backend/dissemination/test_search.py +++ b/backend/dissemination/test_search.py @@ -223,14 +223,43 @@ def test_audit_year(self): results = search_general( audit_years=[2016], ) + assert_all_results_public(self, results) self.assertEqual(len(results), 0) results = search_general( audit_years=[2020], ) + assert_all_results_public(self, results) self.assertEqual(len(results), 1) results = search_general( audit_years=[2020, 2021, 2022], ) + assert_all_results_public(self, results) self.assertEqual(len(results), 3) + + def test_auditee_state(self): + """Given a state, search_general should return only records with a matching auditee_state""" + al = baker.make(General, is_public=True, auditee_state="AL") + baker.make(General, is_public=True, auditee_state="AK") + baker.make( + General, + is_public=True, + auditee_state="AZ", + audit_year="2020", + oversight_agency="01", + auditee_uei="not-looking-for-this-uei", + ) + + # there should be on result for AL + results = search_general(auditee_state="AL") + + assert_all_results_public(self, results) + self.assertEqual(len(results), 1) + self.assertEqual(results[0], al) + + # there should be no results for WI + results = search_general(auditee_state="WI") + + assert_all_results_public(self, results) + self.assertEqual(len(results), 0) diff --git a/backend/dissemination/views.py b/backend/dissemination/views.py index 6cc0aff2d5..8d2bab9bf9 100644 --- a/backend/dissemination/views.py +++ b/backend/dissemination/views.py @@ -9,6 +9,8 @@ from audit.file_downloads import get_download_url, get_filename from audit.models import SingleAuditChecklist +from config.settings import STATE_ABBREVS + from dissemination.forms import SearchForm from dissemination.search import search_general from dissemination.mixins import ReportAccessRequiredMixin @@ -41,7 +43,14 @@ class Search(View): def get(self, request, *args, **kwargs): form = SearchForm() - return render(request, "search.html", {"form": form}) + return render( + request, + "search.html", + { + "form": form, + "state_abbrevs": STATE_ABBREVS, + }, + ) def post(self, request, *args, **kwargs): form = SearchForm(request.POST) @@ -59,6 +68,7 @@ def post(self, request, *args, **kwargs): audit_years = [ int(year) for year in form.cleaned_data["audit_year"] ] # Cast strings from HTML to int + auditee_state = form.cleaned_data["auditee_state"] # TODO: Add a limit choice field to the form limit = form.cleaned_data["limit"] or 30 @@ -77,6 +87,7 @@ def post(self, request, *args, **kwargs): cog_or_oversight=cog_or_oversight, agency_name=agency_name, audit_years=audit_years, + auditee_state=auditee_state, include_private=include_private, ) results_count = results.count() @@ -100,6 +111,7 @@ def post(self, request, *args, **kwargs): context = context | { "form": form, + "state_abbrevs": STATE_ABBREVS, "limit": limit, "results": results, "results_count": results_count,