diff --git a/.github/ISSUE_TEMPLATE/offboarding.md b/.github/ISSUE_TEMPLATE/offboarding.md index b905930b87..6a4b44ec01 100644 --- a/.github/ISSUE_TEMPLATE/offboarding.md +++ b/.github/ISSUE_TEMPLATE/offboarding.md @@ -21,6 +21,8 @@ assignees: '' - [ ] Make a PR to [remove the departing team member from the list of developers and managers](https://github.com/GSA-TTS/FAC/tree/main/terraform/meta/config.tf) with access to our spaces. - [ ] [Remove the departing team member as a member of the FAC group in New Relic.](https://one.newrelic.com/admin-portal/organizations/users-list) (@GSA-TTS/fac-admins can do this) - [ ] If they're leaving TTS altogether, also delete their account. +- [ ] Remove the user from any test accounts (e.g. the Google Group that is used for Cypress test accounts) if they are in that group. +- [ ] Remove from the API GG if they are a member. **For product leads/owners, also...** diff --git a/.github/workflows/failed-data-migration-reprocessor.yml b/.github/workflows/failed-data-migration-reprocessor.yml new file mode 100644 index 0000000000..c9ec4b1483 --- /dev/null +++ b/.github/workflows/failed-data-migration-reprocessor.yml @@ -0,0 +1,50 @@ +--- +name: Failed data migration reprocessor +on: + workflow_dispatch: + inputs: + environment: + required: true + type: choice + description: The environment the workflow should run on. + options: + - dev + - staging + - preview + year: + required: true + type: string + description: Provide audit year. + page_size: + required: true + type: string + description: Number of audit reports by page. + pages: + required: true + type: string + description: Comma-separated list of pages. + error_tag: + required: true + type: string + description: Error tag associated with failed migrations. + +jobs: + historic-data-migrator: + name: Generate and disseminate historic data in ${{ inputs.environment }} database + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + env: + space: ${{ inputs.environment }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run Django command to generate and disseminate historic data in ${{ inputs.environment }} + uses: cloud-gov/cg-cli-tools@main + with: + cf_username: ${{ secrets.CF_USERNAME }} + cf_password: ${{ secrets.CF_PASSWORD }} + cf_org: gsa-tts-oros-fac + cf_space: ${{ env.space }} + command: cf run-task gsa-fac -k 1G -m 1G --name failed_data_migration_reprocessor --command "python manage.py reprocess_failed_migration --year ${{ inputs.year }} --page_size ${{ inputs.page_size }} --pages ${{ inputs.pages }} --error_tag ${{ inputs.error_tag }}" diff --git a/backend/audit/fixtures/basic.pdf b/backend/audit/fixtures/basic.pdf index 3a0f8b7744..f03f1a93a3 100644 Binary files a/backend/audit/fixtures/basic.pdf and b/backend/audit/fixtures/basic.pdf differ diff --git a/backend/audit/validators.py b/backend/audit/validators.py index aa09bd10be..42cf28a488 100644 --- a/backend/audit/validators.py +++ b/backend/audit/validators.py @@ -526,6 +526,8 @@ def validate_single_audit_report_file_extension(file): def validate_pdf_file_integrity(file): """Files must be readable PDFs""" + MIN_CHARARACTERS_IN_PDF = 6000 + try: reader = PdfReader(file) @@ -538,13 +540,13 @@ def validate_pdf_file_integrity(file): for page in reader.pages: page_text = page.extract_text() text_length += len(page_text) - # If we find any characters, we're content. - if text_length > 0: + # If we find enough characters, we're content. + if text_length >= MIN_CHARARACTERS_IN_PDF: break - if text_length == 0: + if text_length < MIN_CHARARACTERS_IN_PDF: raise ValidationError( - "We were unable to process the file you uploaded because it contains no readable text." + "We were unable to process the file you uploaded because it contains no readable text or too little text." ) except ValidationError: diff --git a/backend/audit/views/views.py b/backend/audit/views/views.py index 6047007824..36db6afba4 100644 --- a/backend/audit/views/views.py +++ b/backend/audit/views/views.py @@ -56,6 +56,8 @@ validate_secondary_auditors_json, ) +from dissemination.file_downloads import get_download_url, get_filename + logging.basicConfig( format="%(asctime)s %(levelname)-8s %(module)s:%(lineno)d %(message)s" ) @@ -204,11 +206,28 @@ def _save_audit_data(self, sac, form_section, audit_data): setattr(sac, handler_info["field_name"], audit_data) sac.save() - # this is marked as csrf_exempt to enable by-hand testing via tools like Postman. Should be removed when the frontend form is implemented! @method_decorator(csrf_exempt) def dispatch(self, *args, **kwargs): return super(ExcelFileHandlerView, self).dispatch(*args, **kwargs) + def get(self, request, *args, **kwargs): + """ + Given a report ID and form section, redirect the caller to a download URL for the associated Excel file (if one exists) + """ + try: + report_id = kwargs["report_id"] + form_section = kwargs["form_section"] + + sac = SingleAuditChecklist.objects.get(report_id=report_id) + + filename = get_filename(sac, form_section) + download_url = get_download_url(filename) + + return redirect(download_url) + except SingleAuditChecklist.DoesNotExist as err: + logger.warning("no SingleAuditChecklist found with report ID %s", report_id) + raise PermissionDenied() from err + def post(self, request, *_args, **kwargs): """ Handle Excel file upload: @@ -270,11 +289,6 @@ def post(self, request, *_args, **kwargs): class SingleAuditReportFileHandlerView( SingleAuditChecklistAccessRequiredMixin, generic.View ): - # this is marked as csrf_exempt to enable by-hand testing via tools like Postman. Should be removed when the frontend form is implemented! - @method_decorator(csrf_exempt) - def dispatch(self, *args, **kwargs): - return super(SingleAuditReportFileHandlerView, self).dispatch(*args, **kwargs) - def post(self, request, *args, **kwargs): try: report_id = kwargs["report_id"] diff --git a/backend/census_historical_migration/end_to_end_core.py b/backend/census_historical_migration/end_to_end_core.py index e5f87bc9b8..ea7fd4414c 100644 --- a/backend/census_historical_migration/end_to_end_core.py +++ b/backend/census_historical_migration/end_to_end_core.py @@ -113,44 +113,32 @@ def run_end_to_end(user, audit_header): InspectionRecord.reset() try: sac = setup_sac(user, audit_header) + builder_loader = workbook_builder_loader(user, sac, audit_header) - if sac.general_information["audit_type"] == "alternative-compliance-engagement": - logger.info( - f"Skipping ACE audit: {audit_header.DBKEY} {audit_header.AUDITYEAR}" - ) - raise DataMigrationError( - "Skipping ACE audit", - "skip_ace_audit", - ) - else: - builder_loader = workbook_builder_loader(user, sac, audit_header) - - for section, fun in sections_to_handlers.items(): - builder_loader(fun, section) - - record_dummy_pdf_object(sac, user) - - errors = sac.validate_cross() - - if errors.get("errors"): - raise CrossValidationError( - f"{errors.get('errors')}", "cross_validation" - ) - - step_through_certifications(sac, audit_header) - - disseminate(sac) - - MigrationResult.append_success(f"{sac.report_id} created") - record_migration_status( - audit_header.AUDITYEAR, - audit_header.DBKEY, - ) - record_migration_transformations( - audit_header.AUDITYEAR, - audit_header.DBKEY, - sac.report_id, - ) + for section, fun in sections_to_handlers.items(): + builder_loader(fun, section) + + record_dummy_pdf_object(sac, user) + + errors = sac.validate_cross() + + if errors.get("errors"): + raise CrossValidationError(f"{errors.get('errors')}", "cross_validation") + + step_through_certifications(sac, audit_header) + + disseminate(sac) + + MigrationResult.append_success(f"{sac.report_id} created") + record_migration_status( + audit_header.AUDITYEAR, + audit_header.DBKEY, + ) + record_migration_transformations( + audit_header.AUDITYEAR, + audit_header.DBKEY, + sac.report_id, + ) except Exception as exc: handle_exception(exc, audit_header) diff --git a/backend/census_historical_migration/historic_data_loader.py b/backend/census_historical_migration/historic_data_loader.py index 697d62db24..7f07e319c0 100644 --- a/backend/census_historical_migration/historic_data_loader.py +++ b/backend/census_historical_migration/historic_data_loader.py @@ -27,21 +27,26 @@ def load_historic_data_for_year(audit_year, page_size, pages): logger.info( f"Processing page {page_number} with {page.object_list.count()} submissions." ) + total_count, error_count = perform_migration( + user, page.object_list, total_count, error_count + ) + log_results(error_count, total_count) - for submission in page.object_list: - # Migrate a single submission - run_end_to_end(user, submission) - - MigrationResult.append_summary(submission.AUDITYEAR, submission.DBKEY) - - total_count += 1 - - if MigrationResult.has_errors(): - error_count += 1 - if total_count % 5 == 0: - logger.info(f"Processed = {total_count}, Errors = {error_count}") - log_results(error_count, total_count) +def perform_migration(user, submissions, round_count, total_error_count): + total_count = round_count + error_count = total_error_count + for submission in submissions: + # Migrate a single submission + run_end_to_end(user, submission) + total_count += 1 + if MigrationResult.has_errors(): + error_count += 1 + if total_count % 5 == 0: + logger.info(f"Processed = {total_count}, Errors = {error_count}") + MigrationResult.append_summary(submission.AUDITYEAR, submission.DBKEY) + + return total_count, error_count def log_results(error_count, total_count): diff --git a/backend/census_historical_migration/management/commands/reprocess_failed_migration.py b/backend/census_historical_migration/management/commands/reprocess_failed_migration.py new file mode 100644 index 0000000000..4791c29ba8 --- /dev/null +++ b/backend/census_historical_migration/management/commands/reprocess_failed_migration.py @@ -0,0 +1,47 @@ +from census_historical_migration.process_failed_migration import ( + reprocess_failed_reports, +) +from census_historical_migration.sac_general_lib.utils import ( + normalize_year_string_or_exit, +) + +from django.core.management.base import BaseCommand + +import logging +import sys + + +logger = logging.getLogger(__name__) +logger.setLevel(logging.WARNING) + + +class Command(BaseCommand): + help = """ + Reprocess failed migration reports for a given year and error tag using pagination + Usage: + manage.py run_migration + --year + --page_size + --pages + --error_tag + """ + + def add_arguments(self, parser): + parser.add_argument("--year", help="Audit Year") + parser.add_argument("--page_size", help="Number of records by page", type=int) + parser.add_argument("--pages", help="comma separated pages", type=str) + parser.add_argument("--error_tag", help="error tag", type=str) + + def handle(self, *args, **options): + year = normalize_year_string_or_exit(options.get("year")) + + try: + pages_str = options["pages"] + pages = list(map(lambda d: int(d), pages_str.split(","))) + except ValueError: + logger.error(f"Found a non-integer in pages '{pages_str}'") + sys.exit(-1) + + reprocess_failed_reports( + year, options["page_size"], pages, options["error_tag"] + ) diff --git a/backend/census_historical_migration/process_failed_migration.py b/backend/census_historical_migration/process_failed_migration.py new file mode 100644 index 0000000000..a2f4422183 --- /dev/null +++ b/backend/census_historical_migration/process_failed_migration.py @@ -0,0 +1,56 @@ +import logging + +from .historic_data_loader import ( + create_or_get_user, + log_results, + perform_migration, +) +from .models import ELECAUDITHEADER as AuditHeader, ReportMigrationStatus + +from django.contrib.auth import get_user_model +from django.core.paginator import Paginator + +logger = logging.getLogger(__name__) + +User = get_user_model() + + +def reprocess_failed_reports(audit_year, page_size, pages, error_tag): + """Iterates over and processes submissions for the given audit year""" + total_count = error_count = 0 + user = create_or_get_user() + failed_migrations = ( + ReportMigrationStatus.objects.filter( + audit_year=audit_year, + migration_status="FAILURE", + migrationerrordetail__tag=error_tag, + ) + .order_by("id") + .distinct() + ) + + paginator = Paginator(failed_migrations, page_size) + + logger.info( + f"{failed_migrations.count()} reports have failed migration with error tag {error_tag}" + ) + + for page_number in pages: + if page_number < paginator.num_pages: + page = paginator.page(page_number) + if page.object_list.count() > 0: + dbkey_list = [status.dbkey for status in page.object_list] + + submissions = AuditHeader.objects.filter( + DBKEY__in=dbkey_list, AUDITYEAR=audit_year + ) + logger.info( + f"Processing page {page_number} with {submissions.count() if submissions else 0} submissions." + ) + total_count, error_count = perform_migration( + user, submissions, total_count, error_count + ) + else: + logger.info(f"Skipping page {page_number} as it is out of range") + + log_results(error_count, total_count) diff --git a/backend/census_historical_migration/sac_general_lib/general_information.py b/backend/census_historical_migration/sac_general_lib/general_information.py index f09e14b492..ed9aae4330 100644 --- a/backend/census_historical_migration/sac_general_lib/general_information.py +++ b/backend/census_historical_migration/sac_general_lib/general_information.py @@ -268,7 +268,13 @@ def xform_audit_type(general_information): # Transformation recorded. if general_information.get("audit_type"): value_in_db = general_information["audit_type"] - general_information["audit_type"] = _census_audit_type(value_in_db.upper()) + audit_type = _census_audit_type(value_in_db.upper()) + if audit_type == AUDIT_TYPE_DICT["A"]: + raise DataMigrationError( + "Skipping ACE audit", + "skip_ace_audit", + ) + general_information["audit_type"] = audit_type track_transformations( "AUDITTYPE", value_in_db, diff --git a/backend/census_historical_migration/test_general_information_xforms.py b/backend/census_historical_migration/test_general_information_xforms.py index c1fb1901b7..24230a4a23 100644 --- a/backend/census_historical_migration/test_general_information_xforms.py +++ b/backend/census_historical_migration/test_general_information_xforms.py @@ -199,16 +199,23 @@ def test_missing_period(self): class TestXformAuditType(SimpleTestCase): def test_valid_audit_type(self): for key, value in AUDIT_TYPE_DICT.items(): - with self.subTest(key=key): - general_information = {"audit_type": key} - result = xform_audit_type(general_information) - self.assertEqual(result["audit_type"], value) + if value != "alternative-compliance-engagement": + with self.subTest(key=key): + general_information = {"audit_type": key} + result = xform_audit_type(general_information) + self.assertEqual(result["audit_type"], value) def test_invalid_audit_type(self): general_information = {"audit_type": "invalid_key"} with self.assertRaises(DataMigrationError): xform_audit_type(general_information) + def test_ace_audit_type(self): + # audit type "alternative-compliance-engagement" is not supported at this time. + general_information = {"audit_type": AUDIT_TYPE_DICT["A"]} + with self.assertRaises(DataMigrationError): + xform_audit_type(general_information) + def test_missing_audit_type(self): general_information = {} with self.assertRaises(DataMigrationError): diff --git a/backend/census_historical_migration/throwaway_scripts/generate_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/generate_cli_commands.py deleted file mode 100644 index 7bd6d88de7..0000000000 --- a/backend/census_historical_migration/throwaway_scripts/generate_cli_commands.py +++ /dev/null @@ -1,87 +0,0 @@ -import argparse -import math -import subprocess # nosec - -# This throwaway script spits out code that can be -# copy-pasted into a bash script, or directly into the command line. -# Run from within the Github repo, it should launch instances -# of the historic data migration code. - -parser = argparse.ArgumentParser(description="Print some commands for copy-pasting.") -parser.add_argument("year", type=int, help="Year to run (2022, 2021, etc.)") -parser.add_argument("total_records", type=int, help="Total records.") -parser.add_argument("pages_per_instance", type=int, help="Pages per instance.") -parser.add_argument("instances", type=int, help="How many parallel istances to run") -args = parser.parse_args() - - -# Ceiling division -# We need to always round *up* so we don't miss any records. -def cdiv(a, b): - return math.ceil(a / b) - - -def chunks(lst, n): - """Yield successive n-sized chunks from lst.""" - for i in range(0, len(lst), n): - yield list(map(str, lst[i : i + n])) - - -if __name__ == "__main__": - # The number of pages is the total records // instances - recs_per_instance = cdiv(args.total_records, args.instances) - print(f"Each instance must run {recs_per_instance} records.") - page_size = cdiv(recs_per_instance, args.pages_per_instance) - print( - f"With {args.pages_per_instance} pages per instance, the page size is {page_size}." - ) - total_pages = cdiv(args.total_records, page_size) - print(f"There are {total_pages} pages in total.") - print(f"This means we will attempt {total_pages * page_size} records.") - # Run one extra page for good measure. This "rounds up." - page_chunks = chunks(range(1, total_pages + 1), args.pages_per_instance) - for ndx, page_set in enumerate(page_chunks): - # gh workflow run historic-data-migrator-with-pagination.yml -f environment=preview -f year=2022 -f page_size=1 -f pages=1 - print(f"# Instance {ndx + 1}") - cmd = " ".join( - [ - "gh", - "workflow", - "run", - "historic-data-migrator-with-pagination.yml", - "-f", - "environment=preview", - "-f", - f"year={args.year}", - "-f", - f"page_size={page_size}", - "-f", - "pages={}".format(",".join(page_set)), - ] - ) - print(cmd) - subprocess.run(cmd) # nosec - - -# Examples - -# With round numbers, it comes out nice and tidy. -# python generate_cli_commands.py 2022 42000 5 80 -# Each instance must run 525 records. -# With 5 pages per instance, the page size is 105. -# There are 400 pages in total. -# This means we will attempt 42000 records. - -# Off-by-one, and we make sure we don't drop that extra. -# python generate_cli_commands.py 2022 42001 5 80 -# Each instance must run 526 records. -# With 5 pages per instance, the page size is 106. -# There are 397 pages in total. -# This means we will attempt 42082 records. - -# More pages, and we get closer to the exact number. -# python generate_cli_commands.py 2022 42001 10 80 -# Each instance must run 526 records. -# With 10 pages per instance, the page size is 53. -# There are 793 pages in total. -# This means we will attempt 42029 records. diff --git a/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py new file mode 100644 index 0000000000..c1191e46de --- /dev/null +++ b/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py @@ -0,0 +1,30 @@ +import argparse + +from util import ( + trigger_migration_workflow, +) +import subprocess # nosec + +# This script is a one-off to reprocess mdata migration for a failed +# migration attempt associated with a specific error tag. + +parser = argparse.ArgumentParser( + description="Trigger data migration Github Actions through gh API calls" +) +parser.add_argument("year", type=int, help="Audit year") +parser.add_argument("total_records", type=int, help="Total records.") +parser.add_argument("pages_per_instance", type=int, help="Pages per instance.") +parser.add_argument("instances", type=int, help="How many parallel istances to run") +parser.add_argument("error_tag", type=str, help="Error tag to reprocess") +args = parser.parse_args() + + +if __name__ == "__main__": + cmds = trigger_migration_workflow( + args, workflow_name="failed-data-migration-reprocessor.yml" + ) + print(args) + for cmd in cmds: + cmd.append(f"error_tag={args.error_tag}") + print(" ".join(cmd)) + subprocess.run(cmd) # nosec diff --git a/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py new file mode 100644 index 0000000000..c1adfb8a74 --- /dev/null +++ b/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py @@ -0,0 +1,51 @@ +import argparse +import subprocess # nosec + +from util import trigger_migration_workflow # nosec + +# This throwaway script spits out code that can be +# copy-pasted into a bash script, or directly into the command line. +# Run from within the Github repo, it should launch instances +# of the historic data migration code. + +parser = argparse.ArgumentParser( + description="Trigger data migration Github Actions through gh API calls." +) +parser.add_argument("year", type=int, help="Year to run (2022, 2021, etc.)") +parser.add_argument("total_records", type=int, help="Total records.") +parser.add_argument("pages_per_instance", type=int, help="Pages per instance.") +parser.add_argument("instances", type=int, help="How many parallel istances to run") + +args = parser.parse_args() + + +if __name__ == "__main__": + cmds = trigger_migration_workflow(args) + for cmd in cmds: + cmd = " ".join(cmd) + print(cmd) + subprocess.run(cmd) # nosec + + +# Examples + +# With round numbers, it comes out nice and tidy. +# python generate_cli_commands.py 2022 42000 5 80 +# Each instance must run 525 records. +# With 5 pages per instance, the page size is 105. +# There are 400 pages in total. +# This means we will attempt 42000 records. + +# Off-by-one, and we make sure we don't drop that extra. +# python generate_cli_commands.py 2022 42001 5 80 +# Each instance must run 526 records. +# With 5 pages per instance, the page size is 106. +# There are 397 pages in total. +# This means we will attempt 42082 records. + +# More pages, and we get closer to the exact number. +# python generate_cli_commands.py 2022 42001 10 80 +# Each instance must run 526 records. +# With 10 pages per instance, the page size is 53. +# There are 793 pages in total. +# This means we will attempt 42029 records. diff --git a/backend/census_historical_migration/throwaway_scripts/stop_process_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/stop_process_cli_commands.py index 56ba041572..922adcb574 100644 --- a/backend/census_historical_migration/throwaway_scripts/stop_process_cli_commands.py +++ b/backend/census_historical_migration/throwaway_scripts/stop_process_cli_commands.py @@ -80,7 +80,7 @@ def get_running_task_ids(): f"{id}", ] - print(cmd) + print(" ".join(cmd)) subprocess.run(cmd) # nosec diff --git a/backend/census_historical_migration/throwaway_scripts/util.py b/backend/census_historical_migration/throwaway_scripts/util.py new file mode 100644 index 0000000000..a011e6190c --- /dev/null +++ b/backend/census_historical_migration/throwaway_scripts/util.py @@ -0,0 +1,51 @@ +# Ceiling division +# We need to always round *up* so we don't miss any records. +import math + + +def cdiv(a, b): + return math.ceil(a / b) + + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield list(map(str, lst[i : i + n])) + + +def trigger_migration_workflow( + args, workflow_name="historic-data-migrator-with-pagination.yml" +): + # The number of pages is the total records // instances + recs_per_instance = cdiv(args.total_records, args.instances) + print(f"Each instance must run {recs_per_instance} records.") + page_size = cdiv(recs_per_instance, args.pages_per_instance) + print( + f"With {args.pages_per_instance} pages per instance, the page size is {page_size}." + ) + total_pages = cdiv(args.total_records, page_size) + print(f"There are {total_pages} pages in total.") + print(f"This means we will attempt {total_pages * page_size} records.") + # Run one extra page for good measure. This "rounds up." + page_chunks = chunks(range(1, total_pages + 1), args.pages_per_instance) + cmds = [] + for ndx, page_set in enumerate(page_chunks): + # gh workflow run historic-data-migrator-with-pagination.yml -f environment=preview -f year=2022 -f page_size=1 -f pages=1 + print(f"# Instance {ndx + 1}") + cmds.append( + [ + "gh", + "workflow", + "run", + f"{workflow_name}", + "-f", + "environment=preview", + "-f", + f"year={args.year}", + "-f", + f"page_size={page_size}", + "-f", + "pages={}".format(",".join(page_set)), + ] + ) + return cmds diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index 2d615f51bb..e1197373a9 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -56,7 +56,7 @@ def track_data_transformation(original_value, changed_value, transformation_func ) -def xform_is_minimis_rate_used(rate_content): +def xform_is_minimis_rate_used(rate_content, index=""): """Determines if the de minimis rate was used based on the given text.""" # Transformation recorded. @@ -92,6 +92,15 @@ def xform_is_minimis_rate_used(rate_content): r"utilize(d|s)?\s+(a|an|the)\s+(10|ten)", ] + if index == "1": + track_data_transformation(rate_content, "Y", "xform_is_minimis_rate_used") + return "Y" + elif index == "2": + track_data_transformation(rate_content, "N", "xform_is_minimis_rate_used") + return "N" + elif index == "3": + track_data_transformation(rate_content, "Both", "xform_is_minimis_rate_used") + return "Both" # Check for each pattern in the respective lists for pattern in not_used_patterns: if re.search(pattern, rate_content, re.IGNORECASE): @@ -130,10 +139,12 @@ def _get_minimis_cost_rate(dbkey, year): try: note = Notes.objects.get(DBKEY=dbkey, AUDITYEAR=year, TYPE_ID="2") rate = string_to_string(note.CONTENT) + index = string_to_string(note.NOTE_INDEX) except Notes.DoesNotExist: logger.info(f"De Minimis cost rate not found for dbkey: {dbkey}") rate = "" - return rate + index = "" + return rate, index def _get_notes(dbkey, year): @@ -158,11 +169,13 @@ def generate_notes_to_sefa(audit_header, outfile): uei = xform_retrieve_uei(audit_header.UEI) set_workbook_uei(wb, uei) notes = _get_notes(audit_header.DBKEY, audit_header.AUDITYEAR) - rate_content = _get_minimis_cost_rate(audit_header.DBKEY, audit_header.AUDITYEAR) + rate_content, index = _get_minimis_cost_rate( + audit_header.DBKEY, audit_header.AUDITYEAR + ) policies_content = _get_accounting_policies( audit_header.DBKEY, audit_header.AUDITYEAR ) - is_minimis_rate_used = xform_is_minimis_rate_used(rate_content) + is_minimis_rate_used = xform_is_minimis_rate_used(rate_content, index) set_range(wb, "accounting_policies", [policies_content]) set_range(wb, "is_minimis_rate_used", [is_minimis_rate_used]) diff --git a/backend/cypress/fixtures/basic.pdf b/backend/cypress/fixtures/basic.pdf index 3a0f8b7744..f03f1a93a3 100644 Binary files a/backend/cypress/fixtures/basic.pdf and b/backend/cypress/fixtures/basic.pdf differ diff --git a/backend/report_submission/templates/report_submission/upload-page.html b/backend/report_submission/templates/report_submission/upload-page.html index 3ab14399a4..27bb4f1743 100644 --- a/backend/report_submission/templates/report_submission/upload-page.html +++ b/backend/report_submission/templates/report_submission/upload-page.html @@ -46,6 +46,9 @@

Upload completed worksheet A file has already been uploaded for this section. A successful reupload will overwrite your previous submission.

+

Last uploaded by {{last_uploaded_by}} at {{last_uploaded_at}} Download current workbook

{% endif %} diff --git a/backend/report_submission/views.py b/backend/report_submission/views.py index a45a13599c..e48951310f 100644 --- a/backend/report_submission/views.py +++ b/backend/report_submission/views.py @@ -9,7 +9,10 @@ import api.views +from audit.cross_validation import sac_validation_shape from audit.cross_validation.naming import NC, SECTION_NAMES as SN +from audit.cross_validation.submission_progress_check import section_completed_metadata + from audit.models import Access, SingleAuditChecklist, LateChangeError, SubmissionEvent from audit.validators import validate_general_information_json @@ -299,6 +302,10 @@ def get(self, request, *args, **kwargs): "DB_id": SN[NC.FEDERAL_AWARDS].snake_case, "instructions_url": instructions_base_url + "federal-awards/", "workbook_url": workbook_base_url + "federal-awards-workbook.xlsx", + # below URL handled as a special case because of inconsistent name usage in audit/urls.py and audit/cross_validation/naming.py + "existing_workbook_url": reverse( + "audit:FederalAwardsExpended", args=[report_id] + ), }, "notes-to-sefa": { "view_id": "notes-to-sefa", @@ -307,6 +314,9 @@ def get(self, request, *args, **kwargs): "DB_id": SN[NC.NOTES_TO_SEFA].snake_case, "instructions_url": instructions_base_url + "notes-to-sefa/", "workbook_url": workbook_base_url + "notes-to-sefa-workbook.xlsx", + "existing_workbook_url": reverse( + f"audit:{SN[NC.NOTES_TO_SEFA].camel_case}", args=[report_id] + ), }, "audit-findings": { "view_id": "audit-findings", @@ -318,6 +328,10 @@ def get(self, request, *args, **kwargs): "no_findings_disclaimer": True, "workbook_url": workbook_base_url + "federal-awards-audit-findings-workbook.xlsx", + "existing_workbook_url": reverse( + f"audit:{SN[NC.FINDINGS_UNIFORM_GUIDANCE].camel_case}", + args=[report_id], + ), }, "audit-findings-text": { "view_id": "audit-findings-text", @@ -328,6 +342,9 @@ def get(self, request, *args, **kwargs): + "federal-awards-audit-findings-text/", "no_findings_disclaimer": True, "workbook_url": workbook_base_url + "audit-findings-text-workbook.xlsx", + "existing_workbook_url": reverse( + f"audit:{SN[NC.FINDINGS_TEXT].camel_case}", args=[report_id] + ), }, "cap": { "view_id": "cap", @@ -338,6 +355,10 @@ def get(self, request, *args, **kwargs): "no_findings_disclaimer": True, "workbook_url": workbook_base_url + "corrective-action-plan-workbook.xlsx", + "existing_workbook_url": reverse( + f"audit:{SN[NC.CORRECTIVE_ACTION_PLAN].camel_case}", + args=[report_id], + ), }, "additional-ueis": { "view_id": "additional-ueis", @@ -346,6 +367,9 @@ def get(self, request, *args, **kwargs): "DB_id": SN[NC.ADDITIONAL_UEIS].snake_case, "instructions_url": instructions_base_url + "additional-ueis-workbook/", "workbook_url": workbook_base_url + "additional-ueis-workbook.xlsx", + "existing_workbook_url": reverse( + "audit:AdditionalUeis", args=[report_id] + ), }, "secondary-auditors": { "view_id": "secondary-auditors", @@ -355,6 +379,10 @@ def get(self, request, *args, **kwargs): "instructions_url": instructions_base_url + "secondary-auditors-workbook/", "workbook_url": workbook_base_url + "secondary-auditors-workbook.xlsx", + # below URL handled as a special case because of inconsistent name usage in audit/urls.py and audit/cross_validation/naming.py + "existing_workbook_url": reverse( + f"audit:{SN[NC.SECONDARY_AUDITORS].camel_case}", args=[report_id] + ), }, "additional-eins": { "view_id": "additional-eins", @@ -363,6 +391,10 @@ def get(self, request, *args, **kwargs): "DB_id": SN[NC.ADDITIONAL_EINS].snake_case, "instructions_url": instructions_base_url + "additional-eins-workbook/", "workbook_url": workbook_base_url + "additional-eins-workbook.xlsx", + # below URL handled as a special case because of inconsistent name usage in audit/urls.py and audit/cross_validation/naming.py + "existing_workbook_url": reverse( + "audit:AdditionalEins", args=[report_id] + ), }, } @@ -382,12 +414,20 @@ def get(self, request, *args, **kwargs): } # Using the current URL, append page specific context path_name = request.path.split("/")[2] + for item in additional_context[path_name]: context[item] = additional_context[path_name][item] try: context["already_submitted"] = getattr( sac, additional_context[path_name]["DB_id"] ) + + shaped_sac = sac_validation_shape(sac) + completed_metadata = section_completed_metadata(shaped_sac, path_name) + + context["last_uploaded_by"] = completed_metadata[0] + context["last_uploaded_at"] = completed_metadata[1] + except Exception: context["already_submitted"] = None