Merge pull request #3555 from GSA-TTS/main

2023-03-25 main -> prod

backend/audit/intakelib/intermediate_representation.py

@@ -238,40 +238,51 @@ def is_cell_or_range_coord(s):
     return is_good_range_coord(s) or is_good_cell_coord(s)
 
 
+def process_destination(dn, title, coord, sheets_by_name, workbook):
+    """Process a destination for a defined name."""
+    # Make sure this looks like a range string
+    if is_cell_or_range_coord(coord):
+        range = {}
+        range["name"] = dn.name
+        # If it is a range, grab both parts
+        if is_good_range_coord(coord):
+            range["start_cell"] = abs_ref_to_cell(coord, 0)
+            range["end_cell"] = abs_ref_to_cell(coord, 1)
+        # If it is a single cell (e.g. a UEI cell), then
+        # make the start and end the same
+        elif is_good_cell_coord(coord):
+            range["start_cell"] = abs_ref_to_cell(coord, 0)
+            range["end_cell"] = range["start_cell"]
+
+        # Grab the values for this range using the start/end values
+        range["values"] = load_workbook_range(
+            range["start_cell"], range["end_cell"], workbook[title]
+        )
+
+        # Now, either append to a given sheet, or start a new sheet.
+        if title in sheets_by_name:
+            sheets_by_name[title].append(range)
+        else:
+            sheets_by_name[title] = [range]
+
+
 def extract_workbook_as_ir(file):
     workbook = _open_workbook(file)
     sheets_by_name = {}
+
     for named_range_name in workbook.defined_names:
         dn = workbook.defined_names[named_range_name]
         # If the user mangles the workbook enough, we get #REF errors
         if "#ref" in dn.attr_text.lower():
+            logger.info(f"Workbook has #REF errors for {named_range_name}.")
             raise_modified_workbook(WORKBOOK_MODIFIED_ERROR)
         else:
-            title, coord = next(dn.destinations)
-            # Make sure this looks like a range string
-            if is_cell_or_range_coord(coord):
-                range = {}
-                range["name"] = dn.name
-                # If it is a range, grab both parts
-                if is_good_range_coord(coord):
-                    range["start_cell"] = abs_ref_to_cell(coord, 0)
-                    range["end_cell"] = abs_ref_to_cell(coord, 1)
-                # If it is a single cell (e.g. a UEI cell), then
-                # make the start and end the same
-                elif is_good_cell_coord(coord):
-                    range["start_cell"] = abs_ref_to_cell(coord, 0)
-                    range["end_cell"] = range["start_cell"]
-
-                # Grab the values for this range using the start/end values
-                range["values"] = load_workbook_range(
-                    range["start_cell"], range["end_cell"], workbook[title]
-                )
-
-                # Now, either append to a given sheet, or start a new sheet.
-                if title in sheets_by_name:
-                    sheets_by_name[title].append(range)
-                else:
-                    sheets_by_name[title] = [range]
+            try:
+                title, coord = next(dn.destinations)
+                process_destination(dn, title, coord, sheets_by_name, workbook)
+            except StopIteration:
+                logger.info(f"No destinations found for {named_range_name}.")
+                raise_modified_workbook(WORKBOOK_MODIFIED_ERROR)
 
     # Build the IR, which is a list of sheets.
     sheets = []

backend/audit/test_intakelib.py

@@ -1,6 +1,12 @@
 from django.test import SimpleTestCase
 from copy import deepcopy
-from audit.intakelib.intermediate_representation import ranges_to_rows, remove_null_rows
+from audit.intakelib.intermediate_representation import (
+    extract_workbook_as_ir,
+    ranges_to_rows,
+    remove_null_rows,
+)
+from unittest.mock import MagicMock, patch
+from django.core.exceptions import ValidationError
 
 
 class IRTests(SimpleTestCase):
@@ -79,3 +85,70 @@ def test_remove_null_rows(self):
         cp = deepcopy(IRTests.s2)
         remove_null_rows(cp)
         self.assertEqual(cp, IRTests.r2)
+
+
+class TestExtractWorkbookAsIr(SimpleTestCase):
+    def setUp(self):
+        """Common setup for all tests."""
+        self.destinations = iter([("Sheet1", "$A$1:$A$10")])
+        self.test_range_mock = MagicMock(
+            attr_text="Sheet1!$A$1:$A$10",
+            destinations=self.destinations,
+        )
+        self.test_range_mock.name = "TestRange"
+
+        self.mock_workbook = MagicMock()
+        self.mock_workbook.__getitem__.return_value = MagicMock()  # Default mock sheet
+        self.mock_workbook.defined_names = {"TestRange": self.test_range_mock}
+
+    @patch("audit.intakelib.intermediate_representation._open_workbook")
+    def test_extract_with_valid_input(self, mock_open_workbook):
+        """Test that the function returns the expected intermediate representation."""
+
+        mock_open_workbook.return_value = self.mock_workbook
+
+        # expected result is: [{'name': 'Sheet1', 'ranges': [{'name': 'TestRange', 'start_cell': {'column': 'A', 'row': '1'}, 'end_cell': {'column': 'A', 'row': '0'}, 'values': []}]}]
+        result = extract_workbook_as_ir("dummy_file_path")
+
+        # Assertions
+        mock_open_workbook.assert_called_with("dummy_file_path")
+        self.assertIsInstance(result, list)
+        self.assertGreaterEqual(len(result), 1)
+
+        sheet_info = result[0]
+        self.assertIn("name", sheet_info)
+        self.assertEqual(sheet_info["name"], "Sheet1")
+
+        self.assertIn("ranges", sheet_info)
+        self.assertIsInstance(sheet_info["ranges"], list)
+        self.assertGreaterEqual(len(sheet_info["ranges"]), 1)
+
+        range_info = sheet_info["ranges"][0]
+        self.assertEqual(range_info["name"], "TestRange")
+
+        self.assertEqual(range_info["start_cell"], {"column": "A", "row": "1"})
+        # End cell row is the last cell with a value.
+        # In this case End cell is 0 as there is no value associated with the range
+        self.assertEqual(range_info["end_cell"], {"column": "A", "row": "0"})
+
+        # Validate values list within the range
+        self.assertEqual(range_info["values"], [])
+
+    @patch("audit.intakelib.intermediate_representation._open_workbook")
+    def test_extract_with_ref_error(self, mock_open_workbook):
+        """Test handling of a workbook with a #REF! error."""
+        # Modify test_range_mock for this specific test
+        self.test_range_mock.attr_text = "#REF!"
+        mock_open_workbook.return_value = self.mock_workbook
+
+        with self.assertRaises(ValidationError):
+            extract_workbook_as_ir("dummy_file_with_ref_error")
+
+    @patch("audit.intakelib.intermediate_representation._open_workbook")
+    def test_no_destination_found(self, mock_open_workbook):
+        """Test handling of a workbook with no destinations found for a named range."""
+        self.destinations = iter([])
+        self.test_range_mock.destinations = self.destinations
+        mock_open_workbook.return_value = self.mock_workbook
+        with self.assertRaises(ValidationError):
+            extract_workbook_as_ir("dummy_file_with_malformed_attr_text")

backend/dissemination/summary_reports.py

@@ -528,6 +528,17 @@ def gather_report_data_pre_certification(i2d_data):
 
     data = {}
 
+    # FIXME
+    # This is because additional fields were added to the SF-SAC.
+    # Then, we introduced DisseminationCombined
+    # Then, we got rid of `_` on field names.
+    # Now, this broke.
+    # Choices were made, consequences followed. We want to clean this up.
+    fields_to_ignore = {
+        "federalaward": ["aln"],
+        "finding": ["aln", "federal_agency_prefix", "federal_award_extension"],
+    }
+
     # For every model (FederalAward, CapText, etc), add the skeleton object ('field_names' and empty 'entries') to 'data'.
     # Then, for every object in the dissemination_data (objects of type FederalAward, CapText, etc) create a row (array) for the summary.
     # Every row is created by looping over the field names and appending the data.
@@ -540,7 +551,7 @@ def gather_report_data_pre_certification(i2d_data):
         field_names = [
             field_name
             for field_name in field_name_ordered[model_name]
-            if not field_name.startswith("_")
+            if field_name not in fields_to_ignore.get(model_name, [])
         ]  # Column names, omitting "_" fields
         data[model_name] = {
             "field_names": field_names,