diff --git a/usaspending_api/common/etl/mixins.py b/usaspending_api/common/etl/mixins.py index 8385574036..e5999be59f 100644 --- a/usaspending_api/common/etl/mixins.py +++ b/usaspending_api/common/etl/mixins.py @@ -4,7 +4,7 @@ from usaspending_api.common.etl import ETLObjectBase from usaspending_api.common.etl.operations import delete_obsolete_rows, insert_missing_rows, update_changed_rows from usaspending_api.common.helpers.sql_helpers import execute_dml_sql -from usaspending_api.common.helpers.timing_helpers import ConsoleTimer as Timer +from usaspending_api.common.helpers.timing_helpers import ScriptTimer as Timer class ETLMixin: diff --git a/usaspending_api/references/management/commands/load_gtas.py b/usaspending_api/references/management/commands/load_gtas.py index f7a68bc564..0756731f1c 100644 --- a/usaspending_api/references/management/commands/load_gtas.py +++ b/usaspending_api/references/management/commands/load_gtas.py @@ -41,29 +41,35 @@ class Command(mixins.ETLMixin, BaseCommand): - help = "Update GTAS aggregations used as domain data" + help = "Drop and recreate all GTAS reference data" - @transaction.atomic() def handle(self, *args, **options): - logger.info("Creating broker cursor") + logger.info("Starting ETL script") + self.process_data() + logger.info("GTAS ETL finished successfully!") + + @transaction.atomic() + def process_data(self): broker_cursor = connections["data_broker"].cursor() - logger.info("Running TOTAL_OBLIGATION_SQL") + logger.info("Extracting data from Broker") broker_cursor.execute(self.broker_fetch_sql()) - - logger.info("Getting total obligation values from cursor") total_obligation_values = dictfetchall(broker_cursor) logger.info("Deleting all existing GTAS total obligation records in website") - GTASSF133Balances.objects.all().delete() + deletes = GTASSF133Balances.objects.all().delete() + logger.info(f"Deleted {deletes[0]:,} records") - logger.info("Inserting GTAS total obligations records into website") + logger.info("Transforming new GTAS records") total_obligation_objs = [GTASSF133Balances(**values) for values in total_obligation_values] - GTASSF133Balances.objects.bulk_create(total_obligation_objs) - self._execute_dml_sql(self.tas_fk_sql(), "Populating TAS foreign keys") + logger.info("Loading new GTAS records into database") + new_rec_count = len(GTASSF133Balances.objects.bulk_create(total_obligation_objs)) + logger.info(f"Loaded: {new_rec_count:,} records") - logger.info("GTAS loader finished successfully!") + load_rec = self._execute_dml_sql(self.tas_fk_sql(), "Populating TAS foreign keys") + logger.info(f"Set {load_rec:,} TAS FKs in GTAS table, {new_rec_count - load_rec:,} NULLs") + logger.info("Committing transaction to database") def broker_fetch_sql(self): return f""" @@ -110,7 +116,10 @@ def column_statements(self): return "\n".join(simple_fields + inverted_fields + year_specific_fields) def tas_fk_sql(self): - return """UPDATE gtas_sf133_balances - SET treasury_account_identifier = tas.treasury_account_identifier - FROM treasury_appropriation_account tas - WHERE tas.tas_rendering_label = gtas_sf133_balances.tas_rendering_label;""" + return """ + UPDATE gtas_sf133_balances + SET treasury_account_identifier = tas.treasury_account_identifier + FROM treasury_appropriation_account tas + WHERE + tas.tas_rendering_label = gtas_sf133_balances.tas_rendering_label + AND gtas_sf133_balances.treasury_account_identifier IS DISTINCT FROM tas.treasury_account_identifier"""