Skip to content

Commit

Permalink
fix: applying changes from feedback #2023
Browse files Browse the repository at this point in the history
  • Loading branch information
@dysmento
dysmento committed Nov 9, 2023
1 parent ba632dc commit c300c68
Show file tree
Hide file tree
Showing 6 changed files with 93 additions and 91 deletions.
0 ...erver/src/lib/digest/getGrantsAndUsers.js → ...s/server/src/scripts/getGrantsAndUsers.js
View file
File renamed without changes.
107 changes: 16 additions & 91 deletions terraform/main.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,6 @@ module "arpa_audit_report_security_group" {
allow_all_egress = true
}

module "digest_email_kickoff_cron_security_group" {
source = "cloudposse/security-group/aws"
version = "2.2.0"

namespace = var.namespace
vpc_id = data.aws_ssm_parameter.vpc_id.value
attributes = ["digest_email_kickoff"]
allow_all_egress = true
}

module "digest_email_get_grants_for_criteria_security_group" {
source = "cloudposse/security-group/aws"
version = "2.2.0"
Expand Down Expand Up @@ -351,59 +341,6 @@ resource "aws_iam_role_policy" "api_task-publish_to_arpa_audit_report_queue" {
policy = data.aws_iam_policy_document.publish_to_arpa_audit_report_queue.json
}

module "digest_email_kickoff_cron" {
source = "./modules/scheduled_ecs_task"
enabled = var.enabled && var.enable_digest_email_criteria_cron

name_prefix = "${var.namespace}-digest-email-criteria-cron-"
description = "Executes an ECS task that kicks-off the grants email digest send daily, between 9am - 10am ET."

// Schedule
schedule_expression = "cron(0 9 * * ? *)"
schedule_expression_timezone = "America/New_York"
flexible_time_window = { hours = 1 }
retry_policy_max_attempts = 10
retry_policy_max_event_age = { hours = 4 }

// Permissions
task_role_arn = join("", aws_ecs_task_definition.default.*.task_role_arn)
task_execution_role_arn = join("", aws_ecs_task_definition.default.*.execution_role_arn)
permissions_boundary_arn = var.permissions_boundary_arn

// Task settings
cluster_arn = join("", data.aws_ecs_cluster.default.*.arn)
task_definition_arn = join("", aws_ecs_task_definition.default.*.arn)
task_revision = "LATEST"
launch_type = "FARGATE"
enable_ecs_managed_tags = true
enable_execute_command = false

task_override = jsonencode({
containerOverrides = [
{
name = "api"
command = [
"node",
"-e",
"require('./src/lib/grants-digest').run().then(() => { process.exit(0); }).catch((err) => { console.log(err); process.exit(1); });"
]
environment = [
{
name = "ENABLE_DIGEST_EMAIL_CRITERIA_CRON",
value = "true"
},
]
},
]
})

network_configuration = {
assign_public_ip = false
security_groups = [module.digest_email_kickoff_cron_security_group.id]
subnets = var.subnet_ids
}
}

data "aws_iam_policy_document" "publish_to_digest_email_get_grants" {
statement {
sid = "AllowPublishToQueue"
Expand All @@ -414,7 +351,7 @@ data "aws_iam_policy_document" "publish_to_digest_email_get_grants" {

resource "aws_iam_role_policy" "digest_email_kickoff_cron-publish_to_digest_email_get_grants_queue" {
name_prefix = "send-digest-email-get-grants-requests"
role = module.digest_email_kickoff_cron.ecs_task_role_name
role = module.api.digest_email_kickoff_cron_ecs_task_role_name
policy = data.aws_iam_policy_document.publish_to_digest_email_send_queue.json
}

Expand All @@ -432,31 +369,25 @@ module "digest_email_get_grants" {
docker_tag = var.api_container_image_tag
unified_service_tags = local.unified_service_tags
stop_timeout_seconds = 120
consumer_task_command = ["node", "./src/lib/digest/getGrantsAndUsers.js"]
consumer_task_command = ["node", "./src/scripts/getGrantsAndUsers.js"]
consumer_container_environment = {
API_DOMAIN = local.api_domain_name
DATA_DIR = "/var/data"
NODE_OPTIONS = "--max_old_space_size=3584" # Reserve 512 MB for other task resources
NODE_OPTIONS = "--max_old_space_size=400"
NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}"
WEBSITE_DOMAIN = "https://${var.website_domain_name}"
}
consumer_task_efs_volume_mounts = [{
name = "data"
container_path = "/var/data"
read_only = false
file_system_id = module.api.efs_data_volume_id
access_point_id = module.api.efs_data_volume_access_point_id
}]

# Task resource configuration
# TODO: Tune these values after observing usage in different environments.
# See also: --max_old_space_size in NODE_OPTIONS env var.
consumer_task_size = {
cpu = 1024 # 1 vCPU
memory = 4096 # 4 GB
cpu = 256 # .25 vCPU
memory = 512 # MB
}

# Messaging
autoscaling_message_thresholds = [1, 3, 5, 10, 20, 50]
autoscaling_message_thresholds = [200, 500, 1000, 2000, 5000, 10000]
sqs_publisher = {
principal_type = "Service"
principal_identifier = "ecs-tasks.amazonaws.com"
Expand Down Expand Up @@ -515,13 +446,7 @@ module "digest_email_send" {
NOTIFICATIONS_EMAIL = "grants-notifications@${var.website_domain_name}"
WEBSITE_DOMAIN = "https://${var.website_domain_name}"
}
consumer_task_efs_volume_mounts = [{
name = "data"
container_path = "/var/data"
read_only = false
file_system_id = module.api.efs_data_volume_id
access_point_id = module.api.efs_data_volume_access_point_id
}]

additional_task_role_json_policies = {
send-emails = module.api.send_emails_policy_json
}
Expand All @@ -530,18 +455,18 @@ module "digest_email_send" {
# TODO: Tune these values after observing usage in different environments.
# See also: --max_old_space_size in NODE_OPTIONS env var.
consumer_task_size = {
cpu = 1024 # 1 vCPU
memory = 4096 # 4 GB
cpu = 256 # .25 vCPU
memory = 512 # MB
}

# Messaging
autoscaling_message_thresholds = [1, 3, 5, 10, 20, 50]
autoscaling_message_thresholds = [100, 200, 400, 600, 800, 1000, 5000, 10000]
sqs_publisher = {
principal_type = "Service"
principal_identifier = "ecs-tasks.amazonaws.com"
source_arn = module.api.ecs_service_arn
}
sqs_max_receive_count = 2
sqs_max_receive_count = 1
sqs_dlq_message_retention_seconds = 1209600 # 14 days, in seconds

# Logging
Expand Down Expand Up @@ -656,10 +581,10 @@ module "postgres" {
vpc_id = data.aws_ssm_parameter.vpc_id.value
subnet_ids = local.private_subnet_ids
ingress_security_groups = {
from_api = module.api_to_postgres_security_group.id
from_consume_grants = module.consume_grants_to_postgres_security_group.id
from_arpa_audit_report = module.arpa_audit_report_security_group.id
from_digest_kickoff = module.digest_email_kickoff_cron_security_group.id
from_api = module.api_to_postgres_security_group.id
from_consume_grants = module.consume_grants_to_postgres_security_group.id
from_arpa_audit_report = module.arpa_audit_report_security_group.id
from_digest_kickoff = module.api.digest_email_kickoff_cron_security_group_id
from_digest_get_grants = module.digest_email_get_grants_for_criteria_security_group.id
from_digest_send_email = module.digest_email_send_security_group.id
from_arpa_treasury_report = module.arpa_treasury_report_security_group.id
Expand Down
62 changes: 62 additions & 0 deletions terraform/modules/gost_api/digestcron.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
module "digest_email_kickoff_cron" {
source = "../scheduled_ecs_task"
enabled = var.enabled && var.enable_digest_email_criteria_cron

name_prefix = "${var.namespace}-digest-cron-"
description = "Executes an ECS task that kicks-off the grants email digest send daily, between 9am - 10am ET."

// Schedule
schedule_expression = "cron(0 9 * * ? *)"
schedule_expression_timezone = "America/New_York"
flexible_time_window = { hours = 1 }
retry_policy_max_attempts = 10
retry_policy_max_event_age = { hours = 4 }

// Permissions
task_role_arn = join("", aws_ecs_task_definition.default.*.task_role_arn)
task_execution_role_arn = join("", aws_ecs_task_definition.default.*.execution_role_arn)
permissions_boundary_arn = var.permissions_boundary_arn

// Task settings
cluster_arn = join("", data.aws_ecs_cluster.default.*.arn)
task_definition_arn = join("", aws_ecs_task_definition.default.*.arn)
task_revision = "LATEST"
launch_type = "FARGATE"
enable_ecs_managed_tags = true
enable_execute_command = false

task_override = jsonencode({
containerOverrides = [
{
name = "api"
command = [
"node",
"-e",
"require('./src/lib/grants-digest').run().then(() => { process.exit(0); }).catch((err) => { console.log(err); process.exit(1); });"
]
environment = [
{
name = "ENABLE_DIGEST_EMAIL_CRITERIA_CRON",
value = "true"
},
]
},
]
})

network_configuration = {
assign_public_ip = false
security_groups = [module.digest_email_kickoff_cron_security_group.id]
subnets = var.subnet_ids
}
}

module "digest_email_kickoff_cron_security_group" {
source = "cloudposse/security-group/aws"
version = "2.2.0"

namespace = var.namespace
vpc_id = var.vpc_id
attributes = ["digest_email_kickoff"]
allow_all_egress = true
}
8 changes: 8 additions & 0 deletions terraform/modules/gost_api/outputs.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,11 @@ output "efs_data_volume_access_point_id" {
output "send_emails_policy_json" {
value = module.send_emails_policy.json
}

output "digest_email_kickoff_cron_security_group_id" {
value = module.digest_email_kickoff_cron_security_group.id
}

output "digest_email_kickoff_cron_ecs_task_role_name" {
value = module.digest_email_kickoff_cron.ecs_task_role_name
}
6 changes: 6 additions & 0 deletions terraform/modules/gost_api/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,12 @@ variable "enable_saved_search_grants_digest" {
default = false
}

variable "enable_digest_email_criteria_cron" {
description = "When true, sets the ENABLE_DIGEST_EMAIL_CRITERIA_CRON environment variable to true in the API container."
type = bool
default = false
}

variable "unified_service_tags" {
description = "Datadog unified service tags to apply to runtime environments."
type = object({
Expand Down
1 change: 1 addition & 0 deletions terraform/modules/sqs_consumer_task/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ variable "consumer_task_efs_volume_mounts" {
file_system_id = string
access_point_id = string
}))
default = []
}

variable "stop_timeout_seconds" {
Expand Down

0 comments on commit c300c68

Please sign in to comment.