3695 address historical finding references with invalid formats (#3753)

* #3695 Updated schema and in-file validation rule for workbooks to allow reference number with year >=1900 * #3695 Updated logic to allow reference number with year greater than 1899 * Code improvement. Enforcing reference number check on cap_text and findings_text * #3695 Updated reference number to fail the new schema * #3695 Updated approved versions list
GSA-TTS · Apr 29, 2024 · 1f31913 · 1f31913
1 parent dc47bb5
commit 1f31913
Show file tree

Hide file tree

Showing 32 changed files with 46 additions and 35 deletions.
diff --git a/backend/audit/fixtures/workbooks/should_fail/audit-findings/has_bad_finding_reference.xlsx b/backend/audit/fixtures/workbooks/should_fail/audit-findings/has_bad_finding_reference.xlsx
diff --git a/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py
@@ -14,7 +14,7 @@
 logger = logging.getLogger(__name__)
 
 # A version of these regexes also exists in Base.libsonnet
-PRIOR_REFERENCES_REGEX = r"^20[1-9][0-9]-[0-9]{3}(,\s*20[1-9][0-9]-[0-9]{3})*$"
+PRIOR_REFERENCES_REGEX = r"^[1-2][0-9]{3}-[0-9]{3}(,\s*[1-2][0-9]{3}-[0-9]{3})*$"
 
 
 # DESCRIPTION

diff --git a/backend/audit/intakelib/checks/check_finding_reference_pattern.py b/backend/audit/intakelib/checks/check_finding_reference_pattern.py
@@ -13,12 +13,12 @@
 logger = logging.getLogger(__name__)
 
 # A version of these regexes also exists in Base.libsonnet
-FINDING_REFERENCE_REGEX = r"^20[1-9][0-9]-[0-9]{3}(,\s*20[1-9][0-9]-[0-9]{3})*$"
+FINDING_REFERENCE_REGEX = r"^[1-2][0-9]{3}-[0-9]{3}(,\s*[1-2][0-9]{3}-[0-9]{3})*$"
 
 
 # DESCRIPTION
 # Finding references should be in 20##-### format where the first four
-# digits are a year >= 2010.
+# digits are a year >= 1900.
 # TESTED BY
 # has_bad_references.xlsx
 def finding_reference_pattern(ir):

diff --git a/backend/audit/intakelib/checks/check_version_number.py b/backend/audit/intakelib/checks/check_version_number.py
@@ -7,7 +7,16 @@
 
 logger = logging.getLogger(__name__)
 
-AUTHORIZED_VERSIONS = {"1.0.0", "1.0.1", "1.0.2", "1.0.3", "1.0.4", "1.0.5", "1.1.0"}
+AUTHORIZED_VERSIONS = {
+    "1.0.0",
+    "1.0.1",
+    "1.0.2",
+    "1.0.3",
+    "1.0.4",
+    "1.0.5",
+    "1.1.0",
+    "1.1.1",
+}
 
 
 # DESCRIPTION

diff --git a/backend/audit/intakelib/checks/runners.py b/backend/audit/intakelib/checks/runners.py
@@ -129,13 +129,15 @@
     has_all_the_named_ranges(FORM_SECTIONS.FINDINGS_TEXT),
     has_all_required_fields(FORM_SECTIONS.FINDINGS_TEXT),
     has_invalid_yorn_field(FORM_SECTIONS.FINDINGS_TEXT),
+    finding_reference_pattern,
 ]
 
 corrective_action_plan_checks = general_checks + [
     is_right_workbook(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
     has_all_the_named_ranges(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
     has_all_required_fields(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
     has_invalid_yorn_field(FORM_SECTIONS.CORRECTIVE_ACTION_PLAN),
+    finding_reference_pattern,
 ]
 
 secondary_auditors_checks = general_checks + [

diff --git a/backend/schemas/output/excel/json/additional-eins-workbook.json b/backend/schemas/output/excel/json/additional-eins-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

diff --git a/backend/schemas/output/excel/json/additional-ueis-workbook.json b/backend/schemas/output/excel/json/additional-ueis-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

diff --git a/backend/schemas/output/excel/json/audit-findings-text-workbook.json b/backend/schemas/output/excel/json/audit-findings-text-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {
@@ -102,9 +102,9 @@
                "title_cell": "A1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)",
+                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)",
                   "custom_title": "Reference number",
-                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = \"20\", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
+                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
                   "type": "custom"
                },
                "width": 36

diff --git a/backend/schemas/output/excel/json/corrective-action-plan-workbook.json b/backend/schemas/output/excel/json/corrective-action-plan-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {
@@ -102,9 +102,9 @@
                "title_cell": "A1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)",
+                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)",
                   "custom_title": "Reference number",
-                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = \"20\", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
+                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
                   "type": "custom"
                },
                "width": 36

diff --git a/backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json b/backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {
@@ -118,9 +118,9 @@
                "title_cell": "B1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)",
+                  "custom_error": "Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)",
                   "custom_title": "Reference number",
-                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = \"20\", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
+                  "formula1": "=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = \"-\", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))",
                   "type": "custom"
                },
                "width": 18

diff --git a/backend/schemas/output/excel/json/federal-awards-workbook.json b/backend/schemas/output/excel/json/federal-awards-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

diff --git a/backend/schemas/output/excel/json/notes-to-sefa-workbook.json b/backend/schemas/output/excel/json/notes-to-sefa-workbook.json
@@ -44,7 +44,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

diff --git a/backend/schemas/output/excel/json/secondary-auditors-workbook.json b/backend/schemas/output/excel/json/secondary-auditors-workbook.json
@@ -42,7 +42,7 @@
                "validation": {
                   "type": "NOVALIDATION"
                },
-               "value": "1.0.5",
+               "value": "1.1.1",
                "width": 48
             },
             {

diff --git a/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx b/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx b/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx b/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx b/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx
diff --git a/backend/schemas/output/sections/AdditionalEINs.schema.json b/backend/schemas/output/sections/AdditionalEINs.schema.json
@@ -65,7 +65,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/AdditionalUEIs.schema.json b/backend/schemas/output/sections/AdditionalUEIs.schema.json
@@ -88,7 +88,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/AuditFindingsText.schema.json b/backend/schemas/output/sections/AuditFindingsText.schema.json
@@ -46,7 +46,7 @@
                      },
                      "reference_number": {
                         "description": "Reference Number",
-                        "pattern": "^20[1-9][0-9]-[0-9]{3}$",
+                        "pattern": "^[1-2][0-9]{3}-[0-9]{3}$",
                         "title": "ReferenceNumber",
                         "type": "string"
                      },
@@ -81,7 +81,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/CorrectiveActionPlan.schema.json b/backend/schemas/output/sections/CorrectiveActionPlan.schema.json
@@ -49,7 +49,7 @@
                      },
                      "reference_number": {
                         "description": "Reference Number",
-                        "pattern": "^20[1-9][0-9]-[0-9]{3}$",
+                        "pattern": "^[1-2][0-9]{3}-[0-9]{3}$",
                         "title": "ReferenceNumber",
                         "type": "string"
                      }
@@ -81,7 +81,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/FederalAwards.schema.json b/backend/schemas/output/sections/FederalAwards.schema.json
@@ -552,7 +552,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/FederalAwardsAuditFindings.schema.json b/backend/schemas/output/sections/FederalAwardsAuditFindings.schema.json
@@ -280,7 +280,7 @@
                                        "properties": {
                                           "prior_references": {
                                              "description": "Prior references",
-                                             "pattern": "^20[1-9][0-9]-[0-9]{3}(,\\s*20[1-9][0-9]-[0-9]{3})*$",
+                                             "pattern": "^[1-2][0-9]{3}-[0-9]{3}(,\\s*[1-2][0-9]{3}-[0-9]{3})*$",
                                              "title": "PriorReferences",
                                              "type": "string"
                                           }
@@ -341,7 +341,7 @@
                            },
                            "reference_number": {
                               "description": "Reference Number",
-                              "pattern": "^20[1-9][0-9]-[0-9]{3}$",
+                              "pattern": "^[1-2][0-9]{3}-[0-9]{3}$",
                               "title": "ReferenceNumber",
                               "type": "string"
                            },
@@ -8646,7 +8646,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/NotesToSefa.schema.json b/backend/schemas/output/sections/NotesToSefa.schema.json
@@ -13,7 +13,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/output/sections/SecondaryAuditors.schema.json b/backend/schemas/output/sections/SecondaryAuditors.schema.json
@@ -13,7 +13,7 @@
                "type": "string"
             },
             "version": {
-               "const": "1.1.0",
+               "const": "1.1.1",
                "type": "string"
             }
          },

diff --git a/backend/schemas/source/base/Base.libsonnet b/backend/schemas/source/base/Base.libsonnet
@@ -299,12 +299,12 @@ local Compound = {
   PriorReferences: Types.string {
     title: 'PriorReferences',
     description: 'Prior references',
-    pattern: '^20[1-9][0-9]-[0-9]{3}(,\\s*20[1-9][0-9]-[0-9]{3})*$',
+    pattern: '^[1-2][0-9]{3}-[0-9]{3}(,\\s*[1-2][0-9]{3}-[0-9]{3})*$',
   },
   ReferenceNumber: Types.string {
     title: 'ReferenceNumber',
     description: 'Reference Number',
-    pattern: '^20[1-9][0-9]-[0-9]{3}$',
+    pattern: '^[1-2][0-9]{3}-[0-9]{3}$',
   },
   ComplianceRequirement: {
     title: 'ComplianceRequirement',

diff --git a/backend/schemas/source/excel/libs/SheetValidations.libsonnet b/backend/schemas/source/excel/libs/SheetValidations.libsonnet
@@ -63,8 +63,8 @@ local NumberValidation = {
 local ReferenceNumberValidation = {
   type: 'custom',
   //It is neccessary to allow blank otherwise user cannot delete the value
-  formula1: '=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, LEFT(FIRSTCELLREF, 2) = "20", (MID(FIRSTCELLREF, 3, 1) * 1) > 0, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = "-", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))',
-  custom_error: 'Expecting a value in the format YYYY-NNN, where YYYY is a year after 2010 and NNN a three digit number (e.g. 2023-001)',
+  formula1: '=OR(ISBLANK(FIRSTCELLREF), AND(LEN(FIRSTCELLREF) = 8, (LEFT(FIRSTCELLREF, 2)*1) > 18, ISNUMBER(MID(FIRSTCELLREF, 3, 2) * 1), MID(FIRSTCELLREF, 5, 1) = "-", ISNUMBER(RIGHT(FIRSTCELLREF, 3) * 1)))',
+  custom_error: 'Expecting a value in the format YYYY-NNN, where YYYY is a year and NNN a three digit number (e.g. 2023-001)',
   custom_title: 'Reference number',
 };
 

diff --git a/backend/schemas/source/excel/libs/Sheets.libsonnet b/backend/schemas/source/excel/libs/Sheets.libsonnet
@@ -42,7 +42,7 @@ local section_names = {
 
 // All workbooks should get the same version number.
 // When bumping, also add the new version to AUTHORIZED_VERSIONS in check_version_number.py
-local WORKBOOKS_VERSION = '1.1.0';
+local WORKBOOKS_VERSION = '1.1.1';
 
 {
   single_cell: single_cell,