2509 add a validation check for the sum calculations in federal award…

…s workbook (#2531)

* #2509 Added logic to check cluster total

* #2509 Added logic to check federal program total

* #2509 Added logic to check total amount expended

* #2509 Updated error messages and federal awards check queue

* #2509 Updated federal awards checks

* #2509 Bug fix

* #2509 Regenerated output files

* #2509 Updated test workbooks to include missing cfda_key. The key was absent because these workbooks were auto-generated, not manually created.

---------

Co-authored-by: Dan Swick <[email protected]>

backend/audit/intakelib/checks/check_cluster_total.py

@@ -0,0 +1,75 @@
+import logging
+from audit.intakelib.intermediate_representation import (
+    get_range_values_by_name,
+    get_range_by_name,
+)
+from .util import get_message, build_cell_error_tuple
+
+logger = logging.getLogger(__name__)
+
+STATE_CLUSTER = "STATE CLUSTER"
+OTHER_CLUSTER = "OTHER CLUSTER NOT LISTED ABOVE"
+NOT_APPLICABLE = "N/A"
+
+
+# DESCRIPTION
+# The sum of the amount_expended for a given cluster should equal the corresponding cluster_total
+# K{0}=IF(G{0}="OTHER CLUSTER NOT LISTED ABOVE",SUMIFS(amount_expended,uniform_other_cluster_name,X{0}),IF(AND(OR(G{0}="N/A",G{0}=""),H{0}=""),0,IF(G{0}="STATE CLUSTER",SUMIFS(amount_expended,uniform_state_cluster_name,W{0}),SUMIFS(amount_expended,cluster_name,G{0}))))
+def cluster_total_is_correct(ir):
+    uniform_other_cluster_name = get_range_values_by_name(
+        ir, "uniform_other_cluster_name"
+    )
+    uniform_state_cluster_name = get_range_values_by_name(
+        ir, "uniform_state_cluster_name"
+    )
+    state_cluster_name = get_range_values_by_name(ir, "state_cluster_name")
+    cluster_name = get_range_values_by_name(ir, "cluster_name")
+    cluster_total = get_range_values_by_name(ir, "cluster_total")
+    amount_expended = get_range_values_by_name(ir, "amount_expended")
+
+    errors = []
+
+    # Validating each cluster_total
+    for idx, name in enumerate(cluster_name):
+        # Based on the formula's conditions
+        if name == OTHER_CLUSTER:
+            expected_value = sum(
+                [
+                    amount
+                    for k, amount in zip(uniform_other_cluster_name, amount_expended)
+                    if k == uniform_other_cluster_name[idx]
+                ]
+            )
+        elif (name == NOT_APPLICABLE or not name) and not state_cluster_name[idx]:
+            expected_value = 0
+        elif name == STATE_CLUSTER:
+            expected_value = sum(
+                [
+                    amount
+                    for k, amount in zip(uniform_state_cluster_name, amount_expended)
+                    if k == uniform_state_cluster_name[idx]
+                ]
+            )
+        elif name:
+            expected_value = sum(
+                [
+                    amount
+                    for k, amount in zip(cluster_name, amount_expended)
+                    if k == name
+                ]
+            )
+
+        # Check if the calculated value matches the provided one
+        if expected_value != cluster_total[idx]:
+            errors.append(
+                build_cell_error_tuple(
+                    ir,
+                    get_range_by_name(ir, "cluster_total"),
+                    idx,
+                    get_message("check_cluster_total").format(
+                        cluster_total[idx], expected_value
+                    ),
+                )
+            )
+
+    return errors

backend/audit/intakelib/checks/check_federal_program_total.py

@@ -0,0 +1,39 @@
+import logging
+from audit.intakelib.intermediate_representation import (
+    get_range_values_by_name,
+    get_range_by_name,
+)
+from .util import get_message, build_cell_error_tuple
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# The sum of the amount_expended for a given cfda_key should equal the corresponding federal_program_total
+# J{0}=SUMIFS(amount_expended,cfda_key,V{0}))
+def federal_program_total_is_correct(ir):
+    federal_program_total = get_range_values_by_name(ir, "federal_program_total")
+    cfda_key = get_range_values_by_name(ir, "cfda_key")
+    amount_expended = get_range_values_by_name(ir, "amount_expended")
+
+    errors = []
+
+    # Validating each federal_program_total
+    for idx, key in enumerate(cfda_key):
+        # Compute the sum for current cfda_key
+        computed_sum = sum(
+            [amount for k, amount in zip(cfda_key, amount_expended) if k == key]
+        )
+        if computed_sum != federal_program_total[idx]:
+            errors.append(
+                build_cell_error_tuple(
+                    ir,
+                    get_range_by_name(ir, "federal_program_total"),
+                    idx,
+                    get_message("check_federal_program_total").format(
+                        federal_program_total[idx], computed_sum
+                    ),
+                )
+            )
+
+    return errors

backend/audit/intakelib/checks/check_total_amount_expended.py

@@ -0,0 +1,34 @@
+import logging
+from audit.intakelib.intermediate_representation import (
+    get_range_values_by_name,
+    get_range_by_name,
+)
+from .util import get_message, build_cell_error_tuple
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# The sum of the amount_expended should equal the total_amount_expended
+# B5=SUM(Form!F$2:F$5000)
+def total_amount_expended_is_correct(ir):
+    total_amount_expended_value = get_range_values_by_name(ir, "total_amount_expended")
+    amount_expended_values = get_range_values_by_name(ir, "amount_expended")
+
+    errors = []
+
+    # Validating total_amount_expended
+    computed_sum = sum(amount_expended_values)
+    if computed_sum != total_amount_expended_value[0]:
+        errors.append(
+            build_cell_error_tuple(
+                ir,
+                get_range_by_name(ir, "total_amount_expended"),
+                0,
+                get_message("check_total_amount_expended").format(
+                    total_amount_expended_value[0], computed_sum
+                ),
+            )
+        )
+
+    return errors

backend/audit/intakelib/checks/error_messages.py

@@ -26,6 +26,9 @@
     "check_passthrough_name_when_no_direct_n_and_empty_number": "When the award is direct, passthrough number must be empty",
     "check_findings_grid_validation": "The combination of findings <b>{}</b> is not a valid combination under Uniform Guidance",
     "check_eins_are_not_empty": "EIN cannot be empty",
+    "check_federal_program_total": "Federal program total is {}, but should be {}",
+    "check_cluster_total": "This cluster total is {}, but should be {}",
+    "check_total_amount_expended": "Total amount expended is {}, but should be {}",
     "check_federal_award_amount_passed_through_required": "When Federal Award Passed Through is <b>Y</b>, Amount Passed Through cannot be empty",
     "check_federal_award_amount_passed_through_not_allowed": "When Federal Award Passed Through is <b>N</b>, Amount Passed Through must be empty",
     "check_loan_balance": "The loan balance is currently set to {}. It should either be a positive number, N/A, or left empty.",

backend/audit/intakelib/checks/runners.py

@@ -29,6 +29,9 @@
 from .check_federal_award_passed_always_present import (
     federal_award_passed_always_present,
 )
+from .check_federal_program_total import federal_program_total_is_correct
+from .check_cluster_total import cluster_total_is_correct
+from .check_total_amount_expended import total_amount_expended_is_correct
 from .check_federal_award_passed_passed_through_optional import (
     federal_award_amount_passed_through_optional,
 )
@@ -66,6 +69,9 @@
     no_major_program_no_type,
     all_unique_award_numbers,
     sequential_award_numbers,
+    federal_program_total_is_correct,
+    cluster_total_is_correct,
+    total_amount_expended_is_correct,
     cardinality_of_passthrough_names_and_ids,
 ]
 

backend/schemas/output/excel/json/federal-awards-workbook.json

@@ -458,7 +458,7 @@
             },
             {
                "format": "text",
-               "formula": "=IF(OR(B{0}=\"\",C{0}),\"\",CONCATENATE(B{0},\".\",C{0}))",
+               "formula": "=IF(OR(B{0}=\"\",C{0}=\"\"),\"\",CONCATENATE(B{0},\".\",C{0}))",
                "help": {
                   "link": "https://fac.gov/documentation/validation/#unknown",
                   "text": "Please contact support"

backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet

@@ -283,7 +283,7 @@ local open_ranges_defns = [
   [
     Sheets.open_range {
       keep_locked: true,
-      formula: '=IF(OR(B{0}="",C{0}),"",CONCATENATE(B{0},".",C{0}))',
+      formula: '=IF(OR(B{0}="",C{0}=""),"",CONCATENATE(B{0},".",C{0}))',
       width: 12,
       format: 'text',
       help: Help.unknown,