diff --git a/backend/.profile b/backend/.profile index 3825336b93..2e1e975e5f 100644 --- a/backend/.profile +++ b/backend/.profile @@ -49,6 +49,7 @@ if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then echo 'Finished API schema creation' && echo 'Starting migrate' && python manage.py migrate && + python manage.py migrate --database census-to-gsafac-db && echo 'Finished migrate' && echo 'Starting API view creation' && python manage.py create_api_views && diff --git a/backend/Makefile b/backend/Makefile index 5e2f892ebf..1c1f6b7f00 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -57,7 +57,8 @@ docker-first-run: docker compose build docker compose run web bash -c '\ python manage.py makemigrations &&\ - python manage.py migrate\ + python manage.py migrate &&\ + python manage.py migrate --database census-to-gsafac-db \ ' docker-load-data: @@ -67,6 +68,7 @@ docker-load-data: docker-test: docker compose build docker compose run web python manage.py migrate + docker compose run web python manage.py migrate --database census-to-gsafac-db docker compose run web bash -c 'coverage run --parallel-mode --concurrency=multiprocessing manage.py test --parallel && coverage combine && coverage report -m --fail-under=90' # Run Django tests with docker with no coverage and with scoping @@ -77,6 +79,7 @@ docker-test: docker-nctest: docker compose build docker compose run web python manage.py migrate + docker compose run web python manage.py migrate --database census-to-gsafac-db docker compose run web python manage.py test --parallel ${fac.test.scope} docker-lint: @@ -86,11 +89,13 @@ docker-lint: ghcr-first-run: docker compose -f docker-compose-web.yml run web python manage.py makemigrations docker compose -f docker-compose-web.yml run web python manage.py migrate + docker compose -f docker-compose-web.yml run web python manage.py migrate --database census-to-gsafac-db docker compose -f docker-compose-web.yml run web python manage.py load_test_data # Run Django tests with docker ghcr-test: docker compose -f docker-compose-web.yml run web python manage.py migrate + docker compose -f docker-compose-web.yml run web python manage.py migrate --database census-to-gsafac-db docker compose -f docker-compose-web.yml run web bash -c 'coverage run --parallel-mode --concurrency=multiprocessing manage.py test --parallel && coverage combine && coverage report -m --fail-under=90' # Run Django tests with docker with no coverage and with scoping @@ -100,6 +105,7 @@ ghcr-test: # make nctest ghcr-nctest: docker compose -f docker-compose-web.yml run web python manage.py migrate + docker compose -f docker-compose-web.yml run web python manage.py migrate --database census-to-gsafac-db docker compose -f docker-compose-web.yml run web python manage.py test --parallel ${fac.test.scope} ghcr-lint: diff --git a/backend/census_historical_migration/routers.py b/backend/census_historical_migration/routers.py index 4500e28dcf..534720563c 100644 --- a/backend/census_historical_migration/routers.py +++ b/backend/census_historical_migration/routers.py @@ -1,4 +1,4 @@ -app_name = "census_to_gsafac" +app_name = "census_historical_migration" db_name = "census-to-gsafac-db" @@ -6,12 +6,14 @@ class DBRouter: def db_for_read(self, model, **hints): if model._meta.app_label == app_name: return db_name - return None + return "default" def db_for_write(self, model, **hints): - return self.db_for_read(model, hints) + if model._meta.app_label == app_name: + return db_name + return "default" def allow_migrate(self, db, app_label, model_name=None, **hints): if app_label == app_name: return db == db_name - return False + return db == "default" diff --git a/backend/config/settings.py b/backend/config/settings.py index 7fadbca09b..843267db8c 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -173,10 +173,10 @@ ), "census-to-gsafac-db": env.dj_db_url( "DATABASE_URL_CENSUS_TO_GSAFAC_DB", - default="postgres://postgres:password@0.0.0.0/census-to-gsafac-db", + default="postgres://postgres:password@0.0.0.0:5433/backend", ), } - +DATABASE_ROUTERS = ["census_historical_migration.routers.DBRouter"] POSTGREST = { "URL": env.str("POSTGREST_URL", "http://api:3000"), "LOCAL": env.str("POSTGREST_URL", "http://api:3000"), @@ -230,6 +230,7 @@ STATIC_URL = "/static/" + # Environment specific configurations DEBUG = False if ENVIRONMENT not in ["DEVELOPMENT", "PREVIEW", "STAGING", "PRODUCTION"]: @@ -400,16 +401,10 @@ # Will not be enabled in cloud environments DISABLE_AUTH = False -# Remove once all Census data has been migrated -# Add these as env vars, look at the bucket for values -AWS_CENSUS_ACCESS_KEY_ID = secret("AWS_CENSUS_ACCESS_KEY_ID", "") -AWS_CENSUS_SECRET_ACCESS_KEY = secret("AWS_CENSUS_SECRET_ACCESS_KEY", "") -AWS_CENSUS_STORAGE_BUCKET_NAME = secret("AWS_CENSUS_STORAGE_BUCKET_NAME", "") -AWS_S3_CENSUS_REGION_NAME = secret("AWS_S3_CENSUS_REGION_NAME", "") - ADMIN_URL = "admin/" + # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field diff --git a/backend/config/test_settings.py b/backend/config/test_settings.py new file mode 100644 index 0000000000..3de7f55156 --- /dev/null +++ b/backend/config/test_settings.py @@ -0,0 +1,56 @@ +from django.test import TestCase +from django.conf import settings + + +import boto3 + +from census_historical_migration.models import ELECAUDITHEADER as Gen +from audit.models import SingleAuditChecklist + +CENSUS_DB = [k for k in settings.DATABASES.keys() if "default" != k][0] + + +class SettingsTestCase(TestCase): + databases = {"default", CENSUS_DB} + + def test_models_are_in_appropriate_db(self): + sacs = SingleAuditChecklist.objects.all() + self.assertEqual(len(sacs), 0) + try: + gens = Gen.objects.using("default").all() + except Exception: + self.assertEqual(1, 1) + gens = Gen.objects.using(CENSUS_DB).all() + self.assertEqual(len(gens), 0) + + def test_private_s3(self): + try: + s3_client = boto3.client( + "s3", + aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY, + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + ) + self.assertIsNotNone(s3_client) + items = s3_client.list_objects( + Bucket=settings.AWS_PRIVATE_STORAGE_BUCKET_NAME, + ) + self.assertIsNotNone(items) + except Exception as e: + self.fail(f"Unexpected exception: {e}") + + def test_c2g_s3(self): + try: + s3_client = boto3.client( + "s3", + aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY, + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + ) + self.assertIsNotNone(s3_client) + items = s3_client.list_objects( + Bucket=settings.AWS_CENSUS_TO_GSAFAC_BUCKET_NAME, + ) + self.assertIsNotNone(items) + except Exception as e: + self.fail(f"Unexpected exception: {e}") diff --git a/backend/docker-compose-web.yml b/backend/docker-compose-web.yml index d8e89ffd35..293ba25550 100644 --- a/backend/docker-compose-web.yml +++ b/backend/docker-compose-web.yml @@ -25,7 +25,7 @@ services: ports: - "5433:5432" healthcheck: - test: ["CMD-SHELL", "pg_isready -d postgres -U postgres -p 5433"] + test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"] interval: 10s timeout: 5s retries: 10 @@ -36,12 +36,15 @@ services: depends_on: db: condition: service_healthy + census-to-gsafac-db: + condition: service_healthy minio: condition: service_started clamav-rest: condition: service_started environment: DATABASE_URL: postgres://postgres@db/postgres + DATABASE_URL_CENSUS_TO_GSAFAC_DB: postgres://postgres@census-to-gsafac-db/postgres POSTGREST_URL: http://api:3000 DJANGO_DEBUG: true SAM_API_KEY: ${SAM_API_KEY} diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index b66375f91c..d64892ae0b 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -27,7 +27,7 @@ services: ports: - "5433:5432" healthcheck: - test: ["CMD-SHELL", "pg_isready -d postgres -U postgres -p 5433"] + test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"] interval: 10s timeout: 5s retries: 10 @@ -54,6 +54,8 @@ services: depends_on: db: condition: service_healthy + census-to-gsafac-db: + condition: service_healthy minio: condition: service_started clamav-rest: @@ -62,6 +64,7 @@ services: condition: service_started environment: DATABASE_URL: postgres://postgres@db/postgres + DATABASE_URL_CENSUS_TO_GSAFAC_DB: postgres://postgres@census-to-gsafac-db/postgres POSTGREST_URL: http://api:3000 DJANGO_DEBUG: true SAM_API_KEY: ${SAM_API_KEY} diff --git a/backend/run.sh b/backend/run.sh index 6e2b3e0911..1d428ae0f7 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -20,6 +20,8 @@ fi; # Migrate first python manage.py migrate +python manage.py migrate --database census-to-gsafac-db + echo 'Starting API schema deprecation' && python manage.py drop_deprecated_api_schema_and_views && @@ -30,9 +32,6 @@ echo 'Finished dropping API schema' && echo 'Starting API schema creation' && python manage.py create_api_schema && echo 'Finished API schema creation' && -echo 'Starting migrate' && -python manage.py migrate && -echo 'Finished migrate' && echo 'Starting API view creation' && python manage.py create_api_views && echo 'Finished view creation' && diff --git a/terraform/shared/modules/env/env.tf b/terraform/shared/modules/env/env.tf index 2e8f4158b7..72f3735ad9 100644 --- a/terraform/shared/modules/env/env.tf +++ b/terraform/shared/modules/env/env.tf @@ -8,6 +8,16 @@ module "database" { rds_plan_name = var.database_plan } +module "database-c2g" { + source = "github.com/18f/terraform-cloudgov//database?ref=v0.5.1" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + name = "fac-c2g-db" + recursive_delete = var.recursive_delete + rds_plan_name = var.database_plan +} + module "s3-public" { source = "github.com/18f/terraform-cloudgov//s3?ref=v0.5.1" @@ -28,6 +38,16 @@ module "s3-private" { s3_plan_name = "basic" } +module "s3-c2g" { + source = "github.com/18f/terraform-cloudgov//s3?ref=v0.5.1" + + cf_org_name = var.cf_org_name + cf_space_name = var.cf_space_name + name = "fac-c2g-s3" + recursive_delete = var.recursive_delete + s3_plan_name = "basic" +} + # Stuff used for apps in this space data "cloudfoundry_space" "apps" { org_name = var.cf_org_name @@ -41,4 +61,3 @@ data "cloudfoundry_domain" "public" { data "cloudfoundry_domain" "private" { name = "apps.internal" } -