Merge pull request #3268 from GSA-TTS/main

backend/census_historical_migration/process_failed_migration.py

@@ -36,7 +36,7 @@ def reprocess_failed_reports(audit_year, page_size, pages, error_tag):
     )
 
     for page_number in pages:
-        if page_number < paginator.num_pages:
+        if page_number <= paginator.num_pages:
             page = paginator.page(page_number)
             if page.object_list.count() > 0:
                 dbkey_list = [status.dbkey for status in page.object_list]

backend/census_historical_migration/sac_general_lib/general_information.py

@@ -184,7 +184,9 @@ def _census_audit_type(s):
 def xform_country(general_information, audit_header):
     """Transforms the country from Census format to FAC format."""
     # Transformation to be documented.
-    auditor_country = general_information.get("auditor_country").upper()
+    auditor_country = string_to_string(
+        general_information.get("auditor_country")
+    ).upper()
     if auditor_country in ["US", "USA"]:
         general_information["auditor_country"] = "USA"
     elif auditor_country == "":

backend/census_historical_migration/test_excel_creation.py

@@ -222,3 +222,13 @@ def test_sorting(self):
         self.assertEqual(
             [record.seq_number for record in sorted_records], ["1", "2", "10"]
         )
+
+    def test_sorting_with_empty_field(self):
+        """Test sorting with empty sort field."""
+        records = [
+            self.MockRecord(other="1", seq_number=""),
+            self.MockRecord(other="10", seq_number=""),
+            self.MockRecord(other="2", seq_number=""),
+        ]
+        sorted_records = sort_by_field(records, "seq_number")
+        self.assertEqual([record.other for record in sorted_records], ["1", "10", "2"])

backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py

@@ -1,4 +1,5 @@
 import argparse
+import time
 
 from util import (
     trigger_migration_workflow,
@@ -7,6 +8,8 @@
 
 # This script is a one-off to reprocess mdata migration for a failed
 # migration attempt associated with a specific error tag.
+# Command is `python reprocess_migration_cli_commands.py year total_records pages_per_instance instances error_tag`
+# `python reprocess_migration_cli_commands.py 2022 42000 5 80 invalid_email_error`
 
 parser = argparse.ArgumentParser(
     description="Trigger data migration Github Actions through gh API calls"
@@ -24,7 +27,10 @@
         args, workflow_name="failed-data-migration-reprocessor.yml"
     )
     print(args)
-    for cmd in cmds:
+    for ndx, cmd in enumerate(cmds):
+        print(f"# Instance {ndx + 1}")
+        cmd.append("-f")
         cmd.append(f"error_tag={args.error_tag}")
         print(" ".join(cmd))
         subprocess.run(cmd)  # nosec
+        time.sleep(15)

backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py

@@ -1,7 +1,8 @@
 import argparse
 import subprocess  # nosec
+import time
 
-from util import trigger_migration_workflow  # nosec
+from util import trigger_migration_workflow
 
 # This throwaway script spits out code that can be
 # copy-pasted into a bash script, or directly into the command line.
@@ -21,30 +22,32 @@
 
 if __name__ == "__main__":
     cmds = trigger_migration_workflow(args)
-    for cmd in cmds:
+    for ndx, cmd in enumerate(cmds):
+        print(f"# Instance {ndx + 1}")
         cmd = " ".join(cmd)
         print(cmd)
         subprocess.run(cmd)  # nosec
+        time.sleep(15)
 
 
 # Examples
 
 # With round numbers, it comes out nice and tidy.
-# python generate_cli_commands.py 2022 42000 5 80
+# python start_process_cli_commands.py 2022 42000 5 80
 # Each instance must run 525 records.
 # With 5 pages per instance, the page size is 105.
 # There are 400 pages in total.
 # This means we will attempt 42000 records.
 
 # Off-by-one, and we make sure we don't drop that extra.
-# python generate_cli_commands.py 2022 42001 5 80
+# python start_process_cli_commands.py 2022 42001 5 80
 # Each instance must run 526 records.
 # With 5 pages per instance, the page size is 106.
 # There are 397 pages in total.
 # This means we will attempt 42082 records.
 
 # More pages, and we get closer to the exact number.
-# python generate_cli_commands.py 2022 42001 10 80
+# python start_process_cli_commands.py 2022 42001 10 80
 # Each instance must run 526 records.
 # With 10 pages per instance, the page size is 53.
 # There are 793 pages in total.

backend/census_historical_migration/throwaway_scripts/util.py

@@ -31,7 +31,6 @@ def trigger_migration_workflow(
     cmds = []
     for ndx, page_set in enumerate(page_chunks):
         # gh workflow run historic-data-migrator-with-pagination.yml -f environment=preview -f year=2022 -f page_size=1 -f pages=1
-        print(f"# Instance {ndx + 1}")
         cmds.append(
             [
                 "gh",

backend/census_historical_migration/workbooklib/excel_creation_utils.py

@@ -203,4 +203,4 @@ def sort_by_field(records, sort_field):
     """
     Sorts records by a specified field. The values of the field are converted to integers before sorting.
     """
-    return sorted(records, key=lambda record: int(getattr(record, sort_field)))
+    return sorted(records, key=lambda record: int(getattr(record, sort_field) or 0))