diff --git a/backend/census_historical_migration/test_excel_creation.py b/backend/census_historical_migration/test_excel_creation.py index 8249660733..8ceae113e9 100644 --- a/backend/census_historical_migration/test_excel_creation.py +++ b/backend/census_historical_migration/test_excel_creation.py @@ -222,3 +222,13 @@ def test_sorting(self): self.assertEqual( [record.seq_number for record in sorted_records], ["1", "2", "10"] ) + + def test_sorting_with_empty_field(self): + """Test sorting with empty sort field.""" + records = [ + self.MockRecord(other="1", seq_number=""), + self.MockRecord(other="10", seq_number=""), + self.MockRecord(other="2", seq_number=""), + ] + sorted_records = sort_by_field(records, "seq_number") + self.assertEqual([record.other for record in sorted_records], ["1", "10", "2"]) diff --git a/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py index bc01f858c1..f6e1ae1aac 100644 --- a/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py +++ b/backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py @@ -8,8 +8,8 @@ # This script is a one-off to reprocess mdata migration for a failed # migration attempt associated with a specific error tag. -# Command is `python generate_cli_commands.py year total_records pages_per_instance instances error_tag` -# `python generate_cli_commands.py 2022 42000 5 80 invalid_email_error` +# Command is `python reprocess_migration_cli_commands.py year total_records pages_per_instance instances error_tag` +# `python reprocess_migration_cli_commands.py 2022 42000 5 80 invalid_email_error` parser = argparse.ArgumentParser( description="Trigger data migration Github Actions through gh API calls" diff --git a/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py b/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py index c1adfb8a74..7b8052a26e 100644 --- a/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py +++ b/backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py @@ -1,7 +1,8 @@ import argparse import subprocess # nosec +import time -from util import trigger_migration_workflow # nosec +from util import trigger_migration_workflow # This throwaway script spits out code that can be # copy-pasted into a bash script, or directly into the command line. @@ -21,30 +22,32 @@ if __name__ == "__main__": cmds = trigger_migration_workflow(args) - for cmd in cmds: + for ndx, cmd in enumerate(cmds): + print(f"# Instance {ndx + 1}") cmd = " ".join(cmd) print(cmd) subprocess.run(cmd) # nosec + time.sleep(15) # Examples # With round numbers, it comes out nice and tidy. -# python generate_cli_commands.py 2022 42000 5 80 +# python start_process_cli_commands.py 2022 42000 5 80 # Each instance must run 525 records. # With 5 pages per instance, the page size is 105. # There are 400 pages in total. # This means we will attempt 42000 records. # Off-by-one, and we make sure we don't drop that extra. -# python generate_cli_commands.py 2022 42001 5 80 +# python start_process_cli_commands.py 2022 42001 5 80 # Each instance must run 526 records. # With 5 pages per instance, the page size is 106. # There are 397 pages in total. # This means we will attempt 42082 records. # More pages, and we get closer to the exact number. -# python generate_cli_commands.py 2022 42001 10 80 +# python start_process_cli_commands.py 2022 42001 10 80 # Each instance must run 526 records. # With 10 pages per instance, the page size is 53. # There are 793 pages in total. diff --git a/backend/census_historical_migration/throwaway_scripts/util.py b/backend/census_historical_migration/throwaway_scripts/util.py index a011e6190c..7dce59ffed 100644 --- a/backend/census_historical_migration/throwaway_scripts/util.py +++ b/backend/census_historical_migration/throwaway_scripts/util.py @@ -31,7 +31,6 @@ def trigger_migration_workflow( cmds = [] for ndx, page_set in enumerate(page_chunks): # gh workflow run historic-data-migrator-with-pagination.yml -f environment=preview -f year=2022 -f page_size=1 -f pages=1 - print(f"# Instance {ndx + 1}") cmds.append( [ "gh", diff --git a/backend/census_historical_migration/workbooklib/excel_creation_utils.py b/backend/census_historical_migration/workbooklib/excel_creation_utils.py index 18e647d86c..4be7615e5f 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation_utils.py +++ b/backend/census_historical_migration/workbooklib/excel_creation_utils.py @@ -203,4 +203,4 @@ def sort_by_field(records, sort_field): """ Sorts records by a specified field. The values of the field are converted to integers before sorting. """ - return sorted(records, key=lambda record: int(getattr(record, sort_field))) + return sorted(records, key=lambda record: int(getattr(record, sort_field) or 0))