feat: adds infra required to support digest emails

packages/server/src/lib/digest/grants-digest.js

@@ -0,0 +1,7 @@
+async function run() {
+    console.log('getGrantsAndUsers.run() is called');
+}
+
+module.exports = {
+    run,
+};

packages/server/src/lib/digest/sendEmail.js

@@ -0,0 +1,7 @@
+function main() {
+    console.log('sendEmail.main() is called');
+}
+
+if (require.main === module) {
+    main().then(() => process.exit());
+}

packages/server/src/scripts/digest/cron.js

@@ -0,0 +1,19 @@
+async function run() {
+    console.log('grants-digest-cron run() is called');
+
+    /*
+        The cron is responsible for:
+        1. Identifying all unique criteria from grants_saved_searches
+        2. Constructing a JSON object with the following for each criteria:
+            {
+                criteria: <criteria>,
+                user_ids: [<user_id>, ...],
+                email_date: <email_date>,
+            }
+        3. Publish each object created above as a new message in an SQS queue.
+    */
+}
+
+module.exports = {
+    run,
+};

packages/server/src/scripts/digest/getGrants.js

@@ -0,0 +1,25 @@
+function main() {
+    console.log('getGrants.js.main() is called');
+    /*
+        The getGrants.js script is responsible for:
+        1. Receiving messages from an SQS queue
+        2. Parse the message body into an object with the following shape:
+            {
+                criteria: <criteria>,
+                user_ids: [<user_id>, ...],
+                email_date: <email_date>,
+            }
+        3. For each object, query the grants table for grants that match the criteria
+        4. For each user in user_ids, create a JSON object with the following shape:
+            {
+                user_id: <user_id>,
+                grant_ids: [<grant_id>, ...],
+                email_date: <email_date>,
+            }
+        5. Publish each object created above as a new message in an SQS queue.
+    */
+}
+
+if (require.main === module) {
+    main().then(() => process.exit());
+}

packages/server/src/scripts/digest/sendEmail.js

@@ -0,0 +1,7 @@
+function main() {
+    console.log('sendEmail.main() is called');
+}
+
+if (require.main === module) {
+    main().then(() => process.exit());
+}

packages/server/src/scripts/getGrantsAndUsers.js

@@ -0,0 +1,7 @@
+function main() {
+    console.log('getGrantsAndUsers.js.main() is called');
+}
+
+if (require.main === module) {
+    main().then(() => process.exit());
+}

terraform/main.tf

@@ -115,6 +115,27 @@
   allow_all_egress = true
 }
 
+module "digest_email_get_grants_for_criteria_security_group" {
+  source  = "cloudposse/security-group/aws"
+  version = "2.2.0"
+
+  namespace        = var.namespace
+  vpc_id           = data.aws_ssm_parameter.vpc_id.value
+  attributes       = ["digest_email_get_grants"]
+  allow_all_egress = true
+}
+
+module "digest_email_send_security_group" {
+  source  = "cloudposse/security-group/aws"
+  version = "2.2.0"
+
+  namespace        = var.namespace
+  vpc_id           = data.aws_ssm_parameter.vpc_id.value
+  attributes       = ["digest_email_send"]
+  allow_all_egress = true
+}
+
+
 module "arpa_treasury_report_security_group" {
   source  = "cloudposse/security-group/aws"
   version = "2.2.0"
@@ -338,6 +359,148 @@
   policy      = data.aws_iam_policy_document.publish_to_arpa_audit_report_queue.json
 }
 
+data "aws_iam_policy_document" "publish_to_digest_email_get_grants" {
+  statement {
+    sid       = "AllowPublishToQueue"
+    actions   = ["sqs:SendMessage"]
+    resources = [module.digest_email_get_grants.sqs_queue_arn]
+  }
+}
+
+resource "aws_iam_role_policy" "digest_email_kickoff_cron-publish_to_digest_email_get_grants_queue" {
+  name_prefix = "send-digest-email-get-grants-requests"
+  role        = module.api.ecs_task_role_name
+  policy      = data.aws_iam_policy_document.publish_to_digest_email_send_queue.json
+}
+
+module "digest_email_get_grants" {
+  source                   = "./modules/sqs_consumer_task"
+  namespace                = "${var.namespace}-digest_get_grants"
+  permissions_boundary_arn = local.permissions_boundary_arn
+
+  # Networking
+  subnet_ids         = local.private_subnet_ids
+  security_group_ids = [module.digest_email_get_grants_for_criteria_security_group.id]
+
+  # Task configuration
+  ecs_cluster_name      = join("", aws_ecs_cluster.default.*.name)
+  docker_tag            = var.api_container_image_tag
+  unified_service_tags  = local.unified_service_tags
+  stop_timeout_seconds  = 120
+  consumer_task_command = ["node", "./src/scripts/getGrantsAndUsers.js"]
+  consumer_container_environment = {
+    API_DOMAIN          = local.api_domain_name
+    DATA_DIR            = "/var/data"
+    NODE_OPTIONS        = "--max_old_space_size=400"
+    NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}"
+    WEBSITE_DOMAIN      = "https://${var.website_domain_name}"
+  }
+
+  # Task resource configuration
+  # TODO: Tune these values after observing usage in different environments.
+  #       See also: --max_old_space_size in NODE_OPTIONS env var.
+  consumer_task_size = {
+    cpu    = 256 # .25 vCPU
+    memory = 512 # MB
+  }
+
+  # Messaging
+  autoscaling_message_thresholds = [200, 500, 1000, 2000, 5000, 10000]
+  sqs_publisher = {
+    principal_type       = "Service"
+    principal_identifier = "ecs-tasks.amazonaws.com"
+    source_arn           = module.api.ecs_service_arn
+  }
+  sqs_max_receive_count             = 2
+  sqs_dlq_message_retention_seconds = 1209600 # 14 days, in seconds
+
+  # Logging
+  log_retention = var.api_log_retention_in_days
+
+  # Secrets
+  ssm_path_prefix = var.ssm_service_parameters_path_prefix
+
+  # Postgres
+  rds_db_connect_resources = module.postgres.rds_db_connect_resources_list
+  postgres_username        = module.postgres.master_username
+  postgres_endpoint        = module.postgres.cluster_endpoint
+  postgres_port            = module.postgres.cluster_port
+  postgres_db_name         = module.postgres.default_db_name
+}
+
+data "aws_iam_policy_document" "publish_to_digest_email_send_queue" {
+  statement {
+    sid       = "AllowPublishToQueue"
+    actions   = ["sqs:SendMessage"]
+    resources = [module.digest_email_send.sqs_queue_arn]
+  }
+}
+
+resource "aws_iam_role_policy" "digest_email_get_grants-publish_to_digest_email_send_queue" {
+  name_prefix = "send-digest-email-send-requests"
+  role        = module.digest_email_get_grants.iam_task_role_name
+  policy      = data.aws_iam_policy_document.publish_to_digest_email_send_queue.json
+}
+
+module "digest_email_send" {
+  source                   = "./modules/sqs_consumer_task"
+  namespace                = "${var.namespace}-digest_email_send"
+  permissions_boundary_arn = local.permissions_boundary_arn
+
+  # Networking
+  subnet_ids         = local.private_subnet_ids
+  security_group_ids = [module.digest_email_send_security_group.id]
+
+  # Task configuration
+  ecs_cluster_name      = join("", aws_ecs_cluster.default.*.name)
+  docker_tag            = var.api_container_image_tag
+  unified_service_tags  = local.unified_service_tags
+  stop_timeout_seconds  = 120
+  consumer_task_command = ["node", "./src/lib/digest/sendEmail.js"]
+  consumer_container_environment = {
+    API_DOMAIN          = local.api_domain_name
+    DATA_DIR            = "/var/data"
+    NODE_OPTIONS        = "--max_old_space_size=3584" # Reserve 512 MB for other task resources
+    NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}"
+    WEBSITE_DOMAIN      = "https://${var.website_domain_name}"
+  }
+
+  additional_task_role_json_policies = {
+    send-emails = module.api.send_emails_policy_json
+  }
+
+  # Task resource configuration
+  # TODO: Tune these values after observing usage in different environments.
+  #       See also: --max_old_space_size in NODE_OPTIONS env var.
+  consumer_task_size = {
+    cpu    = 256 # .25 vCPU
+    memory = 512 # MB
+  }
+
+  # Messaging
+  autoscaling_message_thresholds = [100, 200, 400, 600, 800, 1000, 5000, 10000]
+  sqs_publisher = {
+    principal_type       = "Service"
+    principal_identifier = "ecs-tasks.amazonaws.com"
+    source_arn           = module.api.ecs_service_arn
+  }
+  sqs_max_receive_count             = 1
+  sqs_dlq_message_retention_seconds = 1209600 # 14 days, in seconds
+
+  # Logging
+  log_retention = var.api_log_retention_in_days
+
+  # Secrets
+  ssm_path_prefix = var.ssm_service_parameters_path_prefix
+
+  # Postgres
+  rds_db_connect_resources = module.postgres.rds_db_connect_resources_list
+  postgres_username        = module.postgres.master_username
+  postgres_endpoint        = module.postgres.cluster_endpoint
+  postgres_port            = module.postgres.cluster_port
+  postgres_db_name         = module.postgres.default_db_name
+}
+
 module "arpa_treasury_report" {
   source                   = "./modules/sqs_consumer_task"
   namespace                = "${var.namespace}-treasury_report"
@@ -439,6 +602,9 @@
     from_api                  = module.api_to_postgres_security_group.id
     from_consume_grants       = module.consume_grants_to_postgres_security_group.id
     from_arpa_audit_report    = module.arpa_audit_report_security_group.id
+    from_digest_kickoff       = module.api.digest_email_kickoff_cron_security_group_id
+    from_digest_get_grants    = module.digest_email_get_grants_for_criteria_security_group.id
+    from_digest_send_email    = module.digest_email_send_security_group.id
     from_arpa_treasury_report = module.arpa_treasury_report_security_group.id
   }
 

terraform/modules/gost_api/digestcron.tf

@@ -0,0 +1,62 @@
+module "digest_email_kickoff_cron" {
+  source  = "../scheduled_ecs_task"
+  enabled = var.enabled && var.enable_digest_email_criteria_cron
+
+  name_prefix = "${var.namespace}-digest-cron-"
+  description = "Executes an ECS task that kicks-off the grants email digest send daily, between 9am - 10am ET."
+
+  // Schedule
+  schedule_expression          = "cron(0 9 * * ? *)"
+  schedule_expression_timezone = "America/New_York"
+  flexible_time_window         = { hours = 1 }
+  retry_policy_max_attempts    = 10
+  retry_policy_max_event_age   = { hours = 4 }
+
+  // Permissions
+  task_role_arn            = join("", aws_ecs_task_definition.default.*.task_role_arn)
+  task_execution_role_arn  = join("", aws_ecs_task_definition.default.*.execution_role_arn)
+  permissions_boundary_arn = var.permissions_boundary_arn
+
+  // Task settings
+  cluster_arn             = join("", data.aws_ecs_cluster.default.*.arn)
+  task_definition_arn     = join("", aws_ecs_task_definition.default.*.arn)
+  task_revision           = "LATEST"
+  launch_type             = "FARGATE"
+  enable_ecs_managed_tags = true
+  enable_execute_command  = false
+
+  task_override = jsonencode({
+    containerOverrides = [
+      {
+        name = "api"
+        command = [
+          "node",
+          "-e",
+          "require('./src/lib/grants-digest').run().then(() => { process.exit(0); }).catch((err) => { console.log(err); process.exit(1); });"
+        ]
+        environment = [
+          {
+            name  = "ENABLE_DIGEST_EMAIL_CRITERIA_CRON",
+            value = "true"
+          },
+        ]
+      },
+    ]
+  })
+
+  network_configuration = {
+    assign_public_ip = false
+    security_groups  = [module.digest_email_kickoff_cron_security_group.id]
+    subnets          = var.subnet_ids
+  }
+}
+
+module "digest_email_kickoff_cron_security_group" {
+  source  = "cloudposse/security-group/aws"
+  version = "2.2.0"
+
+  namespace        = var.namespace
+  vpc_id           = var.vpc_id
+  attributes       = ["digest_email_kickoff"]
+  allow_all_egress = true
+}

terraform/modules/gost_api/outputs.tf

@@ -47,3 +47,7 @@ output "efs_data_volume_access_point_id" {
 output "send_emails_policy_json" {
   value = module.send_emails_policy.json
 }
+
+output "digest_email_kickoff_cron_security_group_id" {
+  value = module.digest_email_kickoff_cron_security_group.id
+}

terraform/modules/gost_api/variables.tf

@@ -186,6 +186,12 @@ variable "enable_saved_search_grants_digest" {
   default     = false
 }
 
+variable "enable_digest_email_criteria_cron" {
+  description = "When true, sets the ENABLE_DIGEST_EMAIL_CRITERIA_CRON environment variable to true in the API container."
+  type        = bool
+  default     = false
+}
+
 variable "unified_service_tags" {
   description = "Datadog unified service tags to apply to runtime environments."
   type = object({

terraform/modules/sqs_consumer_task/variables.tf

@@ -44,6 +44,7 @@ variable "consumer_task_efs_volume_mounts" {
     file_system_id  = string
     access_point_id = string
   }))
+  default = []
 }
 
 variable "stop_timeout_seconds" {