Use a separate PG DB for Census data (#2573)

* Add bucket and DB

* Lint

* Formatting

* Duplicate changes to compose-web

* Fix naming errors

* Try separate database

* WIp

* Get config test to pass

* Make same changes in docker-compose-web

---------

Co-authored-by: Alex Steel <[email protected]>

backend/.profile

@@ -49,6 +49,7 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
     echo 'Finished API schema creation' &&
     echo 'Starting migrate' &&
     python manage.py migrate &&
+    python manage.py migrate --database census-to-gsafac-db &&
     echo 'Finished migrate' &&
     echo 'Starting API view creation' &&
     python manage.py create_api_views &&

backend/Makefile

@@ -57,7 +57,8 @@ docker-first-run:
 	docker compose build
 	docker compose run web bash -c '\
 		python manage.py makemigrations &&\
-		python manage.py migrate\
+		python manage.py migrate &&\
+		python manage.py migrate --database census-to-gsafac-db \
 		'
 
 docker-load-data:
@@ -67,6 +68,7 @@ docker-load-data:
 docker-test:
 	docker compose build
 	docker compose run web python manage.py migrate
+	docker compose run web python manage.py migrate --database census-to-gsafac-db
 	docker compose run web bash -c 'coverage run --parallel-mode --concurrency=multiprocessing manage.py test --parallel && coverage combine && coverage report -m --fail-under=90'
 
 # Run Django tests with docker with no coverage and with scoping
@@ -77,6 +79,7 @@ docker-test:
 docker-nctest:
 	docker compose build
 	docker compose run web python manage.py migrate
+	docker compose run web python manage.py migrate --database census-to-gsafac-db
 	docker compose run web python manage.py test --parallel ${fac.test.scope}
 
 docker-lint:
@@ -86,11 +89,13 @@ docker-lint:
 ghcr-first-run:
 	docker compose -f docker-compose-web.yml run web python manage.py makemigrations
 	docker compose -f docker-compose-web.yml run web python manage.py migrate
+	docker compose -f docker-compose-web.yml run web python manage.py migrate --database census-to-gsafac-db
 	docker compose -f docker-compose-web.yml run web python manage.py load_test_data
 
 # Run Django tests with docker
 ghcr-test:
 	docker compose -f docker-compose-web.yml run web python manage.py migrate
+	docker compose -f docker-compose-web.yml run web python manage.py migrate --database census-to-gsafac-db
 	docker compose -f docker-compose-web.yml run web bash -c 'coverage run --parallel-mode --concurrency=multiprocessing manage.py test --parallel && coverage combine && coverage report -m --fail-under=90'
 
 # Run Django tests with docker with no coverage and with scoping
@@ -100,6 +105,7 @@ ghcr-test:
 #     make nctest
 ghcr-nctest:
 	docker compose -f docker-compose-web.yml run web python manage.py migrate
+	docker compose -f docker-compose-web.yml run web python manage.py migrate --database census-to-gsafac-db
 	docker compose -f docker-compose-web.yml run web python manage.py test --parallel ${fac.test.scope}
 
 ghcr-lint:

backend/census_historical_migration/routers.py

@@ -1,17 +1,19 @@
-app_name = "census_to_gsafac"
+app_name = "census_historical_migration"
 db_name = "census-to-gsafac-db"
 
 
 class DBRouter:
     def db_for_read(self, model, **hints):
         if model._meta.app_label == app_name:
             return db_name
-        return None
+        return "default"
 
     def db_for_write(self, model, **hints):
-        return self.db_for_read(model, hints)
+        if model._meta.app_label == app_name:
+            return db_name
+        return "default"
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
         if app_label == app_name:
             return db == db_name
-        return False
+        return db == "default"

backend/config/settings.py

@@ -173,10 +173,10 @@
     ),
     "census-to-gsafac-db": env.dj_db_url(
         "DATABASE_URL_CENSUS_TO_GSAFAC_DB",
-        default="postgres://postgres:[email protected]/census-to-gsafac-db",
+        default="postgres://postgres:[email protected]:5433/backend",
     ),
 }
-
+DATABASE_ROUTERS = ["census_historical_migration.routers.DBRouter"]
 POSTGREST = {
     "URL": env.str("POSTGREST_URL", "http://api:3000"),
     "LOCAL": env.str("POSTGREST_URL", "http://api:3000"),
@@ -230,6 +230,7 @@
 
 STATIC_URL = "/static/"
 
+
 # Environment specific configurations
 DEBUG = False
 if ENVIRONMENT not in ["DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]:
@@ -400,16 +401,10 @@
     # Will not be enabled in cloud environments
     DISABLE_AUTH = False
 
-# Remove once all Census data has been migrated
-# Add these as env vars, look at the bucket for values
-AWS_CENSUS_ACCESS_KEY_ID = secret("AWS_CENSUS_ACCESS_KEY_ID", "")
-AWS_CENSUS_SECRET_ACCESS_KEY = secret("AWS_CENSUS_SECRET_ACCESS_KEY", "")
-AWS_CENSUS_STORAGE_BUCKET_NAME = secret("AWS_CENSUS_STORAGE_BUCKET_NAME", "")
-AWS_S3_CENSUS_REGION_NAME = secret("AWS_S3_CENSUS_REGION_NAME", "")
-
 
 ADMIN_URL = "admin/"
 
+
 # Default primary key field type
 # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field
 

backend/config/test_settings.py

@@ -0,0 +1,56 @@
+from django.test import TestCase
+from django.conf import settings
+
+
+import boto3
+
+from census_historical_migration.models import ELECAUDITHEADER as Gen
+from audit.models import SingleAuditChecklist
+
+CENSUS_DB = [k for k in settings.DATABASES.keys() if "default" != k][0]
+
+
+class SettingsTestCase(TestCase):
+    databases = {"default", CENSUS_DB}
+
+    def test_models_are_in_appropriate_db(self):
+        sacs = SingleAuditChecklist.objects.all()
+        self.assertEqual(len(sacs), 0)
+        try:
+            gens = Gen.objects.using("default").all()
+        except Exception:
+            self.assertEqual(1, 1)
+        gens = Gen.objects.using(CENSUS_DB).all()
+        self.assertEqual(len(gens), 0)
+
+    def test_private_s3(self):
+        try:
+            s3_client = boto3.client(
+                "s3",
+                aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+                aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+                endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+            )
+            self.assertIsNotNone(s3_client)
+            items = s3_client.list_objects(
+                Bucket=settings.AWS_PRIVATE_STORAGE_BUCKET_NAME,
+            )
+            self.assertIsNotNone(items)
+        except Exception as e:
+            self.fail(f"Unexpected exception: {e}")
+
+    def test_c2g_s3(self):
+        try:
+            s3_client = boto3.client(
+                "s3",
+                aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID,
+                aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY,
+                endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+            )
+            self.assertIsNotNone(s3_client)
+            items = s3_client.list_objects(
+                Bucket=settings.AWS_CENSUS_TO_GSAFAC_BUCKET_NAME,
+            )
+            self.assertIsNotNone(items)
+        except Exception as e:
+            self.fail(f"Unexpected exception: {e}")

backend/docker-compose-web.yml

@@ -25,7 +25,7 @@ services:
     ports:
       - "5433:5432"
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d postgres -U postgres -p 5433"]
+      test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"]
       interval: 10s
       timeout: 5s
       retries: 10
@@ -36,12 +36,15 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      census-to-gsafac-db:
+        condition: service_healthy
       minio:
         condition: service_started
       clamav-rest:
         condition: service_started
     environment:
       DATABASE_URL: postgres://postgres@db/postgres
+      DATABASE_URL_CENSUS_TO_GSAFAC_DB: postgres://postgres@census-to-gsafac-db/postgres
       POSTGREST_URL: http://api:3000
       DJANGO_DEBUG: true
       SAM_API_KEY: ${SAM_API_KEY}

backend/docker-compose.yml

@@ -27,7 +27,7 @@ services:
     ports:
       - "5433:5432"
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -d postgres -U postgres -p 5433"]
+      test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"]
       interval: 10s
       timeout: 5s
       retries: 10
@@ -54,6 +54,8 @@ services:
     depends_on:
       db:
         condition: service_healthy
+      census-to-gsafac-db:
+        condition: service_healthy
       minio:
         condition: service_started
       clamav-rest:
@@ -62,6 +64,7 @@ services:
         condition: service_started
     environment:
       DATABASE_URL: postgres://postgres@db/postgres
+      DATABASE_URL_CENSUS_TO_GSAFAC_DB: postgres://postgres@census-to-gsafac-db/postgres
       POSTGREST_URL: http://api:3000
       DJANGO_DEBUG: true
       SAM_API_KEY: ${SAM_API_KEY}

backend/run.sh

@@ -20,6 +20,8 @@ fi;
 
 # Migrate first
 python manage.py migrate
+python manage.py migrate --database census-to-gsafac-db
+
 
 echo 'Starting API schema deprecation' &&
 python manage.py drop_deprecated_api_schema_and_views &&
@@ -30,9 +32,6 @@ echo 'Finished dropping API schema' &&
 echo 'Starting API schema creation' &&
 python manage.py create_api_schema &&
 echo 'Finished API schema creation' &&
-echo 'Starting migrate' &&
-python manage.py migrate &&
-echo 'Finished migrate' &&
 echo 'Starting API view creation' &&
 python manage.py create_api_views &&
 echo 'Finished view creation' &&

terraform/shared/modules/env/env.tf

@@ -8,6 +8,16 @@ module "database" {
   rds_plan_name    = var.database_plan
 }
 
+module "database-c2g" {
+  source = "github.com/18f/terraform-cloudgov//database?ref=v0.5.1"
+
+  cf_org_name      = var.cf_org_name
+  cf_space_name    = var.cf_space_name
+  name             = "fac-c2g-db"
+  recursive_delete = var.recursive_delete
+  rds_plan_name    = var.database_plan
+}
+
 module "s3-public" {
   source = "github.com/18f/terraform-cloudgov//s3?ref=v0.5.1"
 
@@ -28,6 +38,16 @@ module "s3-private" {
   s3_plan_name     = "basic"
 }
 
+module "s3-c2g" {
+  source = "github.com/18f/terraform-cloudgov//s3?ref=v0.5.1"
+
+  cf_org_name      = var.cf_org_name
+  cf_space_name    = var.cf_space_name
+  name             = "fac-c2g-s3"
+  recursive_delete = var.recursive_delete
+  s3_plan_name     = "basic"
+}
+
 # Stuff used for apps in this space
 data "cloudfoundry_space" "apps" {
   org_name = var.cf_org_name
@@ -41,4 +61,3 @@ data "cloudfoundry_domain" "public" {
 data "cloudfoundry_domain" "private" {
   name = "apps.internal"
 }
-