diff --git a/packages/server/src/lib/digest/grants-digest.js b/packages/server/src/lib/digest/grants-digest.js new file mode 100644 index 000000000..534f323c0 --- /dev/null +++ b/packages/server/src/lib/digest/grants-digest.js @@ -0,0 +1,7 @@ +async function run() { + console.log('getGrantsAndUsers.run() is called'); +} + +module.exports = { + run, +}; diff --git a/packages/server/src/lib/digest/sendEmail.js b/packages/server/src/lib/digest/sendEmail.js new file mode 100644 index 000000000..9358da0f7 --- /dev/null +++ b/packages/server/src/lib/digest/sendEmail.js @@ -0,0 +1,7 @@ +function main() { + console.log('sendEmail.main() is called'); +} + +if (require.main === module) { + main().then(() => process.exit()); +} diff --git a/packages/server/src/scripts/digest/cron.js b/packages/server/src/scripts/digest/cron.js new file mode 100644 index 000000000..8b7c8e260 --- /dev/null +++ b/packages/server/src/scripts/digest/cron.js @@ -0,0 +1,19 @@ +async function run() { + console.log('grants-digest-cron run() is called'); + + /* + The cron is responsible for: + 1. Identifying all unique criteria from grants_saved_searches + 2. Constructing a JSON object with the following for each criteria: + { + criteria: , + user_ids: [, ...], + email_date: , + } + 3. Publish each object created above as a new message in an SQS queue. + */ +} + +module.exports = { + run, +}; diff --git a/packages/server/src/scripts/digest/getGrants.js b/packages/server/src/scripts/digest/getGrants.js new file mode 100644 index 000000000..9c22e5ec7 --- /dev/null +++ b/packages/server/src/scripts/digest/getGrants.js @@ -0,0 +1,25 @@ +function main() { + console.log('getGrants.js.main() is called'); + /* + The getGrants.js script is responsible for: + 1. Receiving messages from an SQS queue + 2. Parse the message body into an object with the following shape: + { + criteria: , + user_ids: [, ...], + email_date: , + } + 3. For each object, query the grants table for grants that match the criteria + 4. For each user in user_ids, create a JSON object with the following shape: + { + user_id: , + grant_ids: [, ...], + email_date: , + } + 5. Publish each object created above as a new message in an SQS queue. + */ +} + +if (require.main === module) { + main().then(() => process.exit()); +} diff --git a/packages/server/src/scripts/digest/sendEmail.js b/packages/server/src/scripts/digest/sendEmail.js new file mode 100644 index 000000000..9358da0f7 --- /dev/null +++ b/packages/server/src/scripts/digest/sendEmail.js @@ -0,0 +1,7 @@ +function main() { + console.log('sendEmail.main() is called'); +} + +if (require.main === module) { + main().then(() => process.exit()); +} diff --git a/packages/server/src/scripts/getGrantsAndUsers.js b/packages/server/src/scripts/getGrantsAndUsers.js new file mode 100644 index 000000000..58d0dc39d --- /dev/null +++ b/packages/server/src/scripts/getGrantsAndUsers.js @@ -0,0 +1,7 @@ +function main() { + console.log('getGrantsAndUsers.js.main() is called'); +} + +if (require.main === module) { + main().then(() => process.exit()); +} diff --git a/terraform/main.tf b/terraform/main.tf index a8acedd4f..d6f2c894e 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -115,6 +115,27 @@ module "arpa_audit_report_security_group" { allow_all_egress = true } +module "digest_email_get_grants_for_criteria_security_group" { + source = "cloudposse/security-group/aws" + version = "2.2.0" + + namespace = var.namespace + vpc_id = data.aws_ssm_parameter.vpc_id.value + attributes = ["digest_email_get_grants"] + allow_all_egress = true +} + +module "digest_email_send_security_group" { + source = "cloudposse/security-group/aws" + version = "2.2.0" + + namespace = var.namespace + vpc_id = data.aws_ssm_parameter.vpc_id.value + attributes = ["digest_email_send"] + allow_all_egress = true +} + + module "arpa_treasury_report_security_group" { source = "cloudposse/security-group/aws" version = "2.2.0" @@ -338,6 +359,148 @@ resource "aws_iam_role_policy" "api_task-publish_to_arpa_audit_report_queue" { policy = data.aws_iam_policy_document.publish_to_arpa_audit_report_queue.json } +data "aws_iam_policy_document" "publish_to_digest_email_get_grants" { + statement { + sid = "AllowPublishToQueue" + actions = ["sqs:SendMessage"] + resources = [module.digest_email_get_grants.sqs_queue_arn] + } +} + +resource "aws_iam_role_policy" "digest_email_kickoff_cron-publish_to_digest_email_get_grants_queue" { + name_prefix = "send-digest-email-get-grants-requests" + role = module.api.ecs_task_role_name + policy = data.aws_iam_policy_document.publish_to_digest_email_send_queue.json +} + +module "digest_email_get_grants" { + source = "./modules/sqs_consumer_task" + namespace = "${var.namespace}-digest_get_grants" + permissions_boundary_arn = local.permissions_boundary_arn + + # Networking + subnet_ids = local.private_subnet_ids + security_group_ids = [module.digest_email_get_grants_for_criteria_security_group.id] + + # Task configuration + ecs_cluster_name = join("", aws_ecs_cluster.default.*.name) + docker_tag = var.api_container_image_tag + unified_service_tags = local.unified_service_tags + stop_timeout_seconds = 120 + consumer_task_command = ["node", "./src/scripts/getGrantsAndUsers.js"] + consumer_container_environment = { + API_DOMAIN = local.api_domain_name + DATA_DIR = "/var/data" + NODE_OPTIONS = "--max_old_space_size=400" + NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}" + WEBSITE_DOMAIN = "https://${var.website_domain_name}" + } + + # Task resource configuration + # TODO: Tune these values after observing usage in different environments. + # See also: --max_old_space_size in NODE_OPTIONS env var. + consumer_task_size = { + cpu = 256 # .25 vCPU + memory = 512 # MB + } + + # Messaging + autoscaling_message_thresholds = [200, 500, 1000, 2000, 5000, 10000] + sqs_publisher = { + principal_type = "Service" + principal_identifier = "ecs-tasks.amazonaws.com" + source_arn = module.api.ecs_service_arn + } + sqs_max_receive_count = 2 + sqs_dlq_message_retention_seconds = 1209600 # 14 days, in seconds + + # Logging + log_retention = var.api_log_retention_in_days + + # Secrets + ssm_path_prefix = var.ssm_service_parameters_path_prefix + + # Postgres + rds_db_connect_resources = module.postgres.rds_db_connect_resources_list + postgres_username = module.postgres.master_username + postgres_endpoint = module.postgres.cluster_endpoint + postgres_port = module.postgres.cluster_port + postgres_db_name = module.postgres.default_db_name +} + +data "aws_iam_policy_document" "publish_to_digest_email_send_queue" { + statement { + sid = "AllowPublishToQueue" + actions = ["sqs:SendMessage"] + resources = [module.digest_email_send.sqs_queue_arn] + } +} + +resource "aws_iam_role_policy" "digest_email_get_grants-publish_to_digest_email_send_queue" { + name_prefix = "send-digest-email-send-requests" + role = module.digest_email_get_grants.iam_task_role_name + policy = data.aws_iam_policy_document.publish_to_digest_email_send_queue.json +} + +module "digest_email_send" { + source = "./modules/sqs_consumer_task" + namespace = "${var.namespace}-digest_email_send" + permissions_boundary_arn = local.permissions_boundary_arn + + # Networking + subnet_ids = local.private_subnet_ids + security_group_ids = [module.digest_email_send_security_group.id] + + # Task configuration + ecs_cluster_name = join("", aws_ecs_cluster.default.*.name) + docker_tag = var.api_container_image_tag + unified_service_tags = local.unified_service_tags + stop_timeout_seconds = 120 + consumer_task_command = ["node", "./src/lib/digest/sendEmail.js"] + consumer_container_environment = { + API_DOMAIN = local.api_domain_name + DATA_DIR = "/var/data" + NODE_OPTIONS = "--max_old_space_size=3584" # Reserve 512 MB for other task resources + NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}" + WEBSITE_DOMAIN = "https://${var.website_domain_name}" + } + + additional_task_role_json_policies = { + send-emails = module.api.send_emails_policy_json + } + + # Task resource configuration + # TODO: Tune these values after observing usage in different environments. + # See also: --max_old_space_size in NODE_OPTIONS env var. + consumer_task_size = { + cpu = 256 # .25 vCPU + memory = 512 # MB + } + + # Messaging + autoscaling_message_thresholds = [100, 200, 400, 600, 800, 1000, 5000, 10000] + sqs_publisher = { + principal_type = "Service" + principal_identifier = "ecs-tasks.amazonaws.com" + source_arn = module.api.ecs_service_arn + } + sqs_max_receive_count = 1 + sqs_dlq_message_retention_seconds = 1209600 # 14 days, in seconds + + # Logging + log_retention = var.api_log_retention_in_days + + # Secrets + ssm_path_prefix = var.ssm_service_parameters_path_prefix + + # Postgres + rds_db_connect_resources = module.postgres.rds_db_connect_resources_list + postgres_username = module.postgres.master_username + postgres_endpoint = module.postgres.cluster_endpoint + postgres_port = module.postgres.cluster_port + postgres_db_name = module.postgres.default_db_name +} + module "arpa_treasury_report" { source = "./modules/sqs_consumer_task" namespace = "${var.namespace}-treasury_report" @@ -439,6 +602,9 @@ module "postgres" { from_api = module.api_to_postgres_security_group.id from_consume_grants = module.consume_grants_to_postgres_security_group.id from_arpa_audit_report = module.arpa_audit_report_security_group.id + from_digest_kickoff = module.api.digest_email_kickoff_cron_security_group_id + from_digest_get_grants = module.digest_email_get_grants_for_criteria_security_group.id + from_digest_send_email = module.digest_email_send_security_group.id from_arpa_treasury_report = module.arpa_treasury_report_security_group.id } diff --git a/terraform/modules/gost_api/digestcron.tf b/terraform/modules/gost_api/digestcron.tf new file mode 100644 index 000000000..34ff485b6 --- /dev/null +++ b/terraform/modules/gost_api/digestcron.tf @@ -0,0 +1,62 @@ +module "digest_email_kickoff_cron" { + source = "../scheduled_ecs_task" + enabled = var.enabled && var.enable_digest_email_criteria_cron + + name_prefix = "${var.namespace}-digest-cron-" + description = "Executes an ECS task that kicks-off the grants email digest send daily, between 9am - 10am ET." + + // Schedule + schedule_expression = "cron(0 9 * * ? *)" + schedule_expression_timezone = "America/New_York" + flexible_time_window = { hours = 1 } + retry_policy_max_attempts = 10 + retry_policy_max_event_age = { hours = 4 } + + // Permissions + task_role_arn = join("", aws_ecs_task_definition.default.*.task_role_arn) + task_execution_role_arn = join("", aws_ecs_task_definition.default.*.execution_role_arn) + permissions_boundary_arn = var.permissions_boundary_arn + + // Task settings + cluster_arn = join("", data.aws_ecs_cluster.default.*.arn) + task_definition_arn = join("", aws_ecs_task_definition.default.*.arn) + task_revision = "LATEST" + launch_type = "FARGATE" + enable_ecs_managed_tags = true + enable_execute_command = false + + task_override = jsonencode({ + containerOverrides = [ + { + name = "api" + command = [ + "node", + "-e", + "require('./src/lib/grants-digest').run().then(() => { process.exit(0); }).catch((err) => { console.log(err); process.exit(1); });" + ] + environment = [ + { + name = "ENABLE_DIGEST_EMAIL_CRITERIA_CRON", + value = "true" + }, + ] + }, + ] + }) + + network_configuration = { + assign_public_ip = false + security_groups = [module.digest_email_kickoff_cron_security_group.id] + subnets = var.subnet_ids + } +} + +module "digest_email_kickoff_cron_security_group" { + source = "cloudposse/security-group/aws" + version = "2.2.0" + + namespace = var.namespace + vpc_id = var.vpc_id + attributes = ["digest_email_kickoff"] + allow_all_egress = true +} diff --git a/terraform/modules/gost_api/outputs.tf b/terraform/modules/gost_api/outputs.tf index 7f2777286..93176bd17 100644 --- a/terraform/modules/gost_api/outputs.tf +++ b/terraform/modules/gost_api/outputs.tf @@ -47,3 +47,7 @@ output "efs_data_volume_access_point_id" { output "send_emails_policy_json" { value = module.send_emails_policy.json } + +output "digest_email_kickoff_cron_security_group_id" { + value = module.digest_email_kickoff_cron_security_group.id +} \ No newline at end of file diff --git a/terraform/modules/gost_api/variables.tf b/terraform/modules/gost_api/variables.tf index a9175e828..f7f709c2e 100644 --- a/terraform/modules/gost_api/variables.tf +++ b/terraform/modules/gost_api/variables.tf @@ -186,6 +186,12 @@ variable "enable_saved_search_grants_digest" { default = false } +variable "enable_digest_email_criteria_cron" { + description = "When true, sets the ENABLE_DIGEST_EMAIL_CRITERIA_CRON environment variable to true in the API container." + type = bool + default = false +} + variable "unified_service_tags" { description = "Datadog unified service tags to apply to runtime environments." type = object({ diff --git a/terraform/modules/sqs_consumer_task/variables.tf b/terraform/modules/sqs_consumer_task/variables.tf index 06ce96f70..07a1c90d2 100644 --- a/terraform/modules/sqs_consumer_task/variables.tf +++ b/terraform/modules/sqs_consumer_task/variables.tf @@ -44,6 +44,7 @@ variable "consumer_task_efs_volume_mounts" { file_system_id = string access_point_id = string })) + default = [] } variable "stop_timeout_seconds" {