Make import_eoni_from_s3 command report to slack

DemocracyClub · Nov 27, 2024 · bf3868d · bf3868d
1 parent 80163b9
commit bf3868d
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 6 deletions.
diff --git a/deploy/files/scripts/import_eoni_from_s3.sh b/deploy/files/scripts/import_eoni_from_s3.sh
@@ -2,14 +2,24 @@
 set -e
 set -x
 
-DC_ENVIRONMENT=$DC_ENVIRONMENT
+if [ -z "$DC_ENVIRONMENT" ]; then
+   echo "DC_ENVIRONMENT is not set"
+   exit 1
+fi
+
 BUCKET_NAME="eoni-data.wheredoivote.co.uk.${DC_ENVIRONMENT}"
 SRCDIR='/tmp/eoni_production_data'
 PREM_4326_CSV=${SRCDIR}/PREM_4326.csv
 PRO_4326_CSV=${SRCDIR}/PRO_4326.csv
 
 LATEST_FILE=$(/usr/local/bin/aws s3 ls s3://"${BUCKET_NAME}/" | sort | tail -n1 | rev | cut -d' ' -f1 | rev)
 
+if [ "$DC_ENVIRONMENT" = "production" ]; then
+   SLACK_CHANNEL="bots"
+else
+   SLACK_CHANNEL="bot-testing"
+fi
+
 rm -rf $SRCDIR && mkdir -p $SRCDIR
 
 /usr/local/bin/aws s3 cp s3://"${BUCKET_NAME}/${LATEST_FILE}" $SRCDIR
@@ -26,6 +36,6 @@ mlr --icsv --otsv --headerless-csv-output cut -f PRO_X_COR,PRO_Y_COR $SRCDIR/"${
 
 paste -d ',' ${PRO_4326_CSV} ${PREM_4326_CSV} $SRCDIR/"${LATEST_FILE}" > eoni_reprojected.csv
 
-/usr/bin/manage-py-command import_eoni --cleanup --reprojected eoni_reprojected.csv
+/usr/bin/manage-py-command import_eoni --cleanup --reprojected --slack ${SLACK_CHANNEL} eoni_reprojected.csv
 
 rm ${PREM_4326_CSV} ${PRO_4326_CSV} eoni_reprojected.csv
diff --git a/polling_stations/apps/data_importers/management/commands/import_eoni.py b/polling_stations/apps/data_importers/management/commands/import_eoni.py
@@ -90,10 +90,9 @@ def handle(self, *args, **options):
             }
             self.stations_only = options.get("stations_only")
             self.pre_process_data(reprojected=options["reprojected"])
-            with transaction.atomic():
-                self.clear_old_data()
-                self.copy_data()
-                self.assign_uprn_to_councils()
+            self.clear_old_data()
+            self.copy_data()
+            self.assign_uprn_to_councils()
             super().handle(*args, **options)
             if options.get("cleanup"):
                 [path.unlink() for path in self.paths.values() if path.exists()]