Keep LAST_IMPORT_SHA on master for development env

This should fix and issue we were having where dev deploys would 'fail'
CI because the 'LAST_IMPORT_SHA' in parameter store had been updated to
a value which wasn't in the tree anymore, meaning we couldn't git diff
against it.

cdk/lambdas/wdiv-s3-trigger/tests/test_handler.py

@@ -22,7 +22,7 @@
     {
       "eventVersion": "2.0",
       "eventSource": "aws:s3",
-      "awsRegion": "eu-west-1",
+      "awsRegion": "eu-west-2",
       "eventTime": "1970-01-01T00:00:00.000Z",
       "eventName": "ObjectCreated:Put",
       "userIdentity": {
@@ -59,7 +59,7 @@
 
 ios_payload = json.loads(trigger_payload_str.replace("X01000000", "IOS"))
 
-os.environ["AWS_DEFAULT_REGION"] = moto_s3_responses.DEFAULT_REGION_NAME = "eu-west-1"
+os.environ["AWS_DEFAULT_REGION"] = moto_s3_responses.DEFAULT_REGION_NAME = "eu-west-2"
 
 
 @pytest.mark.django_db
@@ -90,7 +90,7 @@ def setUp(self):
         self.repo = "chris48s/does-not-exist"
         self.upload_bucket = "fake-upload-bucket"
         self.final_bucket = "fake-final-bucket"
-        region = "eu-west-1"
+        region = "eu-west-2"
         os.environ["GITHUB_REPO"] = self.repo
         os.environ["GITHUB_API_KEY"] = "testing"
         os.environ["WDIV_API_KEY"] = "testing"

polling_stations/apps/data_importers/management/commands/run_new_imports.py

@@ -5,6 +5,7 @@
 import boto3
 import botocore
 from django.apps import apps
+from django.conf import settings
 from django.core.management import BaseCommand, call_command
 
 
@@ -41,9 +42,30 @@ def any_non_import_scripts(changed):
     return any(not is_import_script(path) for path in changed)
 
 
+def sha_in_tree(sha):
+    try:
+        git_rev_parse(sha)
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+
+class LastImportShaNotInTreeError(Exception):
+    pass
+
+
 def get_last_import_sha_from_ssm():
     ssm_client = boto3.client("ssm")
     response = ssm_client.get_parameter(Name="LAST_IMPORT_SHA")
+    last_import_sha = response["Parameter"]["Value"]
+    if not sha_in_tree(last_import_sha) and settings.DC_ENVIRONMENT == "development":
+        # We often end up with an out of tree sha in parameter store on development.
+        # Just set 'from_sha' to master
+        return git_rev_parse("master")
+    if not sha_in_tree(last_import_sha):
+        raise LastImportShaNotInTreeError(
+            f"Value of LAST_IMPORT_SHA ('{last_import_sha}') stored in parameter store not in working tree."
+        )
     return response["Parameter"]["Value"]
 
 
@@ -107,7 +129,7 @@ def output_summary(self):
             else:
                 self.stdout.write(line[1])
 
-    def update_last_import_sha_on_ssm(self, to_sha):
+    def update_last_import_sha_on_ssm(self, to_sha, dc_environment=None):
         self.stdout.write("Updating LAST_IMPORT_SHA on ssm...")
         try:
             ssm_client = boto3.client("ssm")
@@ -153,6 +175,7 @@ def handle(self, *args, **options):
         is_post_deploy = options.get("post_deploy")
 
         changed_paths = get_paths_changed(from_sha, to_sha)
+
         changed_scripts = get_changed_scripts(changed_paths)
         has_imports = any_import_scripts(changed_paths)
         has_application = any_non_import_scripts(changed_paths)

polling_stations/apps/data_importers/tests/test_run_new_imports.py

@@ -1,17 +1,23 @@
+import os
 import re
 from io import StringIO
 from unittest.mock import Mock, patch
 
+import pytest
+
 from data_importers.management.commands.run_new_imports import (
     any_import_scripts,
     any_non_import_scripts,
     get_changed_scripts,
     get_paths_changed,
     git_rev_parse,
     is_import_script,
+    sha_in_tree,
+    get_last_import_sha_from_ssm,
+    LastImportShaNotInTreeError,
 )
 from django.core.management import call_command
-from django.test import TestCase
+from django.test import TestCase, override_settings
 
 no_scripts = [
     "polling_stations/apps/councils/management/commands/import_councils.py",
@@ -30,6 +36,9 @@
     "requirements/base.txt",
 ]
 
+os.environ["AWS_DEFAULT_REGION"] = "eu-west-2"
+IN_TREE_SHA = "1012c398b13d2e9c87f718b87e07ee9cd1c26222"
+
 
 def test_get_paths_changed():
     expected_empty = []
@@ -48,7 +57,7 @@ def test_get_paths_changed():
     ]
     # This commit sha needs to be in the repos history
     assert expected_empty == get_paths_changed(
-        "1012c398b13d2e9c87f718b87e07ee9cd1c26222",
+        IN_TREE_SHA,
         "1012c398b13d2e9c87f718b87e07ee9cd1c26222",
     )
 
@@ -85,6 +94,11 @@ def test_is_import_script():
     )
 
 
+def test_sha_in_tree():
+    assert sha_in_tree(IN_TREE_SHA)
+    assert not sha_in_tree("not-in-tree")
+
+
 def test_get_changed_scripts():
     changed = [
         "polling_stations/apps/councils/management/commands/import_councils.py",
@@ -114,6 +128,74 @@ def test_any_non_import_scripts():
     assert any_non_import_scripts(mix_of_scripts_and_not_scripts)
 
 
+@pytest.fixture
+def mock_ssm():
+    with patch("boto3.client") as mock_client:
+        mock_ssm = Mock()
+        mock_client.return_value = mock_ssm
+        yield mock_ssm
+
+
+def test_get_last_import_sha_success(mock_ssm):
+    mock_ssm.get_parameter.return_value = {"Parameter": {"Value": IN_TREE_SHA}}
+
+    result = get_last_import_sha_from_ssm()
+
+    assert result == IN_TREE_SHA
+    mock_ssm.get_parameter.assert_called_once_with(Name="LAST_IMPORT_SHA")
+
+
+def test_get_last_import_sha_failure(mock_ssm):
+    not_in_tree_sha = "notintree"
+    mock_ssm.get_parameter.return_value = {"Parameter": {"Value": not_in_tree_sha}}
+
+    with pytest.raises(LastImportShaNotInTreeError) as err:
+        get_last_import_sha_from_ssm()
+
+    assert (
+        str(err.value)
+        == f"Value of LAST_IMPORT_SHA ('{not_in_tree_sha}') stored in parameter store not in working tree."
+    )
+    mock_ssm.get_parameter.assert_called_once_with(Name="LAST_IMPORT_SHA")
+
+
+@override_settings(DC_ENVIRONMENT="production")
+def test_get_last_import_sha_failure_prod(mock_ssm):
+    not_in_tree_sha = "notintree"
+    mock_ssm.get_parameter.return_value = {"Parameter": {"Value": not_in_tree_sha}}
+
+    with pytest.raises(LastImportShaNotInTreeError) as err:
+        get_last_import_sha_from_ssm()
+
+    assert (
+        str(err.value)
+        == f"Value of LAST_IMPORT_SHA ('{not_in_tree_sha}') stored in parameter store not in working tree."
+    )
+    mock_ssm.get_parameter.assert_called_once_with(Name="LAST_IMPORT_SHA")
+
+
+@override_settings(DC_ENVIRONMENT="development")
+def test_get_last_import_sha_not_in_tree_dev(mock_ssm):
+    not_in_tree_sha = "notintree"
+    master_sha = "headofmasterbranch"
+    mock_ssm.get_parameter.return_value = {"Parameter": {"Value": not_in_tree_sha}}
+
+    with patch(
+        "data_importers.management.commands.run_new_imports.sha_in_tree"
+    ) as mock_sha_in_tree, patch(
+        "data_importers.management.commands.run_new_imports.git_rev_parse"
+    ) as mock_git_rev_parse:
+        # First call returns False (original SHA not in tree), second call returns True (master SHA is in tree)
+        mock_sha_in_tree.side_effect = [False, True]
+        # Just mock rev_parse to always return master
+        mock_git_rev_parse.return_value = master_sha
+
+        result = get_last_import_sha_from_ssm()
+
+    assert result == master_sha
+    mock_ssm.get_parameter.assert_called_once_with(Name="LAST_IMPORT_SHA")
+
+
 class test_run_new_imports(TestCase):
     maxDiff = 1000
     run_scripts_mock = Mock()

polling_stations/settings/base.py

@@ -331,7 +331,7 @@ def get_ec2_ip():
 )
 
 # When we're running on AWS
-if os.environ.get("DC_ENVIRONMENT"):
+if DC_ENVIRONMENT := os.environ.get("DC_ENVIRONMENT", None):
     if not Path(INITIAL_REPLICATION_COMPLETE_FILE).exists():
         DATABASES["local"] = {
             "ENGINE": "django.contrib.gis.db.backends.postgis",