diff --git a/packages/server/src/lib/digest/getGrantsAndUsers.js b/packages/server/src/scripts/getGrantsAndUsers.js similarity index 100% rename from packages/server/src/lib/digest/getGrantsAndUsers.js rename to packages/server/src/scripts/getGrantsAndUsers.js diff --git a/terraform/main.tf b/terraform/main.tf index 89d2a5064..fedcbb100 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -99,16 +99,6 @@ module "arpa_audit_report_security_group" { allow_all_egress = true } -module "digest_email_kickoff_cron_security_group" { - source = "cloudposse/security-group/aws" - version = "2.2.0" - - namespace = var.namespace - vpc_id = data.aws_ssm_parameter.vpc_id.value - attributes = ["digest_email_kickoff"] - allow_all_egress = true -} - module "digest_email_get_grants_for_criteria_security_group" { source = "cloudposse/security-group/aws" version = "2.2.0" @@ -351,59 +341,6 @@ resource "aws_iam_role_policy" "api_task-publish_to_arpa_audit_report_queue" { policy = data.aws_iam_policy_document.publish_to_arpa_audit_report_queue.json } -module "digest_email_kickoff_cron" { - source = "./modules/scheduled_ecs_task" - enabled = var.enabled && var.enable_digest_email_criteria_cron - - name_prefix = "${var.namespace}-digest-email-criteria-cron-" - description = "Executes an ECS task that kicks-off the grants email digest send daily, between 9am - 10am ET." - - // Schedule - schedule_expression = "cron(0 9 * * ? *)" - schedule_expression_timezone = "America/New_York" - flexible_time_window = { hours = 1 } - retry_policy_max_attempts = 10 - retry_policy_max_event_age = { hours = 4 } - - // Permissions - task_role_arn = join("", aws_ecs_task_definition.default.*.task_role_arn) - task_execution_role_arn = join("", aws_ecs_task_definition.default.*.execution_role_arn) - permissions_boundary_arn = var.permissions_boundary_arn - - // Task settings - cluster_arn = join("", data.aws_ecs_cluster.default.*.arn) - task_definition_arn = join("", aws_ecs_task_definition.default.*.arn) - task_revision = "LATEST" - launch_type = "FARGATE" - enable_ecs_managed_tags = true - enable_execute_command = false - - task_override = jsonencode({ - containerOverrides = [ - { - name = "api" - command = [ - "node", - "-e", - "require('./src/lib/grants-digest').run().then(() => { process.exit(0); }).catch((err) => { console.log(err); process.exit(1); });" - ] - environment = [ - { - name = "ENABLE_DIGEST_EMAIL_CRITERIA_CRON", - value = "true" - }, - ] - }, - ] - }) - - network_configuration = { - assign_public_ip = false - security_groups = [module.digest_email_kickoff_cron_security_group.id] - subnets = var.subnet_ids - } -} - data "aws_iam_policy_document" "publish_to_digest_email_get_grants" { statement { sid = "AllowPublishToQueue" @@ -414,7 +351,7 @@ data "aws_iam_policy_document" "publish_to_digest_email_get_grants" { resource "aws_iam_role_policy" "digest_email_kickoff_cron-publish_to_digest_email_get_grants_queue" { name_prefix = "send-digest-email-get-grants-requests" - role = module.digest_email_kickoff_cron.ecs_task_role_name + role = module.api.digest_email_kickoff_cron_ecs_task_role_name policy = data.aws_iam_policy_document.publish_to_digest_email_send_queue.json } @@ -432,31 +369,25 @@ module "digest_email_get_grants" { docker_tag = var.api_container_image_tag unified_service_tags = local.unified_service_tags stop_timeout_seconds = 120 - consumer_task_command = ["node", "./src/lib/digest/getGrantsAndUsers.js"] + consumer_task_command = ["node", "./src/scripts/getGrantsAndUsers.js"] consumer_container_environment = { API_DOMAIN = local.api_domain_name DATA_DIR = "/var/data" - NODE_OPTIONS = "--max_old_space_size=3584" # Reserve 512 MB for other task resources + NODE_OPTIONS = "--max_old_space_size=400" NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}" WEBSITE_DOMAIN = "https://${var.website_domain_name}" } - consumer_task_efs_volume_mounts = [{ - name = "data" - container_path = "/var/data" - read_only = false - file_system_id = module.api.efs_data_volume_id - access_point_id = module.api.efs_data_volume_access_point_id - }] + # Task resource configuration # TODO: Tune these values after observing usage in different environments. # See also: --max_old_space_size in NODE_OPTIONS env var. consumer_task_size = { - cpu = 1024 # 1 vCPU - memory = 4096 # 4 GB + cpu = 256 # .25 vCPU + memory = 512 # MB } # Messaging - autoscaling_message_thresholds = [1, 3, 5, 10, 20, 50] + autoscaling_message_thresholds = [200, 500, 1000, 2000, 5000, 10000] sqs_publisher = { principal_type = "Service" principal_identifier = "ecs-tasks.amazonaws.com" @@ -515,13 +446,7 @@ module "digest_email_send" { NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}" WEBSITE_DOMAIN = "https://${var.website_domain_name}" } - consumer_task_efs_volume_mounts = [{ - name = "data" - container_path = "/var/data" - read_only = false - file_system_id = module.api.efs_data_volume_id - access_point_id = module.api.efs_data_volume_access_point_id - }] + additional_task_role_json_policies = { send-emails = module.api.send_emails_policy_json } @@ -530,18 +455,18 @@ module "digest_email_send" { # TODO: Tune these values after observing usage in different environments. # See also: --max_old_space_size in NODE_OPTIONS env var. consumer_task_size = { - cpu = 1024 # 1 vCPU - memory = 4096 # 4 GB + cpu = 256 # .25 vCPU + memory = 512 # MB } # Messaging - autoscaling_message_thresholds = [1, 3, 5, 10, 20, 50] + autoscaling_message_thresholds = [100, 200, 400, 600, 800, 1000, 5000, 10000] sqs_publisher = { principal_type = "Service" principal_identifier = "ecs-tasks.amazonaws.com" source_arn = module.api.ecs_service_arn } - sqs_max_receive_count = 2 + sqs_max_receive_count = 1 sqs_dlq_message_retention_seconds = 1209600 # 14 days, in seconds # Logging @@ -656,10 +581,10 @@ module "postgres" { vpc_id = data.aws_ssm_parameter.vpc_id.value subnet_ids = local.private_subnet_ids ingress_security_groups = { - from_api = module.api_to_postgres_security_group.id - from_consume_grants = module.consume_grants_to_postgres_security_group.id - from_arpa_audit_report = module.arpa_audit_report_security_group.id - from_digest_kickoff = module.digest_email_kickoff_cron_security_group.id + from_api = module.api_to_postgres_security_group.id + from_consume_grants = module.consume_grants_to_postgres_security_group.id + from_arpa_audit_report = module.arpa_audit_report_security_group.id + from_digest_kickoff = module.api.digest_email_kickoff_cron_security_group_id from_digest_get_grants = module.digest_email_get_grants_for_criteria_security_group.id from_digest_send_email = module.digest_email_send_security_group.id from_arpa_treasury_report = module.arpa_treasury_report_security_group.id diff --git a/terraform/modules/gost_api/digestcron.tf b/terraform/modules/gost_api/digestcron.tf new file mode 100644 index 000000000..34ff485b6 --- /dev/null +++ b/terraform/modules/gost_api/digestcron.tf @@ -0,0 +1,62 @@ +module "digest_email_kickoff_cron" { + source = "../scheduled_ecs_task" + enabled = var.enabled && var.enable_digest_email_criteria_cron + + name_prefix = "${var.namespace}-digest-cron-" + description = "Executes an ECS task that kicks-off the grants email digest send daily, between 9am - 10am ET." + + // Schedule + schedule_expression = "cron(0 9 * * ? *)" + schedule_expression_timezone = "America/New_York" + flexible_time_window = { hours = 1 } + retry_policy_max_attempts = 10 + retry_policy_max_event_age = { hours = 4 } + + // Permissions + task_role_arn = join("", aws_ecs_task_definition.default.*.task_role_arn) + task_execution_role_arn = join("", aws_ecs_task_definition.default.*.execution_role_arn) + permissions_boundary_arn = var.permissions_boundary_arn + + // Task settings + cluster_arn = join("", data.aws_ecs_cluster.default.*.arn) + task_definition_arn = join("", aws_ecs_task_definition.default.*.arn) + task_revision = "LATEST" + launch_type = "FARGATE" + enable_ecs_managed_tags = true + enable_execute_command = false + + task_override = jsonencode({ + containerOverrides = [ + { + name = "api" + command = [ + "node", + "-e", + "require('./src/lib/grants-digest').run().then(() => { process.exit(0); }).catch((err) => { console.log(err); process.exit(1); });" + ] + environment = [ + { + name = "ENABLE_DIGEST_EMAIL_CRITERIA_CRON", + value = "true" + }, + ] + }, + ] + }) + + network_configuration = { + assign_public_ip = false + security_groups = [module.digest_email_kickoff_cron_security_group.id] + subnets = var.subnet_ids + } +} + +module "digest_email_kickoff_cron_security_group" { + source = "cloudposse/security-group/aws" + version = "2.2.0" + + namespace = var.namespace + vpc_id = var.vpc_id + attributes = ["digest_email_kickoff"] + allow_all_egress = true +} diff --git a/terraform/modules/gost_api/outputs.tf b/terraform/modules/gost_api/outputs.tf index 1ec9f1772..59df7dcc3 100644 --- a/terraform/modules/gost_api/outputs.tf +++ b/terraform/modules/gost_api/outputs.tf @@ -47,3 +47,11 @@ output "efs_data_volume_access_point_id" { output "send_emails_policy_json" { value = module.send_emails_policy.json } + +output "digest_email_kickoff_cron_security_group_id" { + value = module.digest_email_kickoff_cron_security_group.id +} + +output "digest_email_kickoff_cron_ecs_task_role_name" { + value = module.digest_email_kickoff_cron.ecs_task_role_name +} \ No newline at end of file diff --git a/terraform/modules/gost_api/variables.tf b/terraform/modules/gost_api/variables.tf index 235b62cf3..8005d86b4 100644 --- a/terraform/modules/gost_api/variables.tf +++ b/terraform/modules/gost_api/variables.tf @@ -174,6 +174,12 @@ variable "enable_saved_search_grants_digest" { default = false } +variable "enable_digest_email_criteria_cron" { + description = "When true, sets the ENABLE_DIGEST_EMAIL_CRITERIA_CRON environment variable to true in the API container." + type = bool + default = false +} + variable "unified_service_tags" { description = "Datadog unified service tags to apply to runtime environments." type = object({ diff --git a/terraform/modules/sqs_consumer_task/variables.tf b/terraform/modules/sqs_consumer_task/variables.tf index 06ce96f70..07a1c90d2 100644 --- a/terraform/modules/sqs_consumer_task/variables.tf +++ b/terraform/modules/sqs_consumer_task/variables.tf @@ -44,6 +44,7 @@ variable "consumer_task_efs_volume_mounts" { file_system_id = string access_point_id = string })) + default = [] } variable "stop_timeout_seconds" {