diff --git a/.github/workflows/terraform-apply-env.yml b/.github/workflows/terraform-apply-env.yml index 810f708071..2e8160095f 100644 --- a/.github/workflows/terraform-apply-env.yml +++ b/.github/workflows/terraform-apply-env.yml @@ -22,6 +22,8 @@ jobs: TF_VAR_cf_user: ${{ secrets.CF_USERNAME }} TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }} TF_VAR_new_relic_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + TF_VAR_new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + TF_VAR_new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }} TF_VAR_pgrst_jwt_secret: ${{ secrets.PGRST_JWT_SECRET }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TERRAFORM_PRE_RUN: | diff --git a/.github/workflows/terraform-plan-env.yml b/.github/workflows/terraform-plan-env.yml index e503eed63a..a52e084c07 100644 --- a/.github/workflows/terraform-plan-env.yml +++ b/.github/workflows/terraform-plan-env.yml @@ -17,6 +17,8 @@ jobs: TF_VAR_cf_user: ${{ secrets.CF_USERNAME }} TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }} TF_VAR_new_relic_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + TF_VAR_new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }} + TF_VAR_new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }} TF_VAR_pgrst_jwt_secret: ${{ secrets.PGRST_JWT_SECRET }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TERRAFORM_PRE_RUN: | diff --git a/terraform/dev/dev.tf b/terraform/dev/dev.tf index d206f0ee76..6067a043cd 100644 --- a/terraform/dev/dev.tf +++ b/terraform/dev/dev.tf @@ -2,6 +2,8 @@ module "dev" { source = "../shared/modules/env" cf_space_name = "dev" new_relic_license_key = var.new_relic_license_key + new_relic_account_id = var.new_relic_account_id + new_relic_api_key = var.new_relic_api_key pgrst_jwt_secret = var.pgrst_jwt_secret database_plan = "medium-gp-psql" diff --git a/terraform/dev/variables.tf b/terraform/dev/variables.tf index fe8367f7e1..5dc957c792 100644 --- a/terraform/dev/variables.tf +++ b/terraform/dev/variables.tf @@ -3,6 +3,16 @@ variable "new_relic_license_key" { description = "the license key to use when setting up the New Relic agent" } +variable "new_relic_account_id" { + type = number + description = "New Relic Account ID" +} + +variable "new_relic_api_key" { + type = string + description = "New Relic API key" +} + variable "pgrst_jwt_secret" { type = string description = "the JWT signing secret for validating JWT tokens from api.data.gov" diff --git a/terraform/preview/preview.tf b/terraform/preview/preview.tf index 6cb6bb6871..5bc758ebb4 100644 --- a/terraform/preview/preview.tf +++ b/terraform/preview/preview.tf @@ -2,6 +2,8 @@ module "preview" { source = "../shared/modules/env" cf_space_name = "preview" new_relic_license_key = var.new_relic_license_key + new_relic_account_id = var.new_relic_account_id + new_relic_api_key = var.new_relic_api_key pgrst_jwt_secret = var.pgrst_jwt_secret database_plan = "medium-gp-psql" diff --git a/terraform/preview/variables.tf b/terraform/preview/variables.tf index 06ed1df495..2d752a8002 100644 --- a/terraform/preview/variables.tf +++ b/terraform/preview/variables.tf @@ -3,6 +3,16 @@ variable "new_relic_license_key" { description = "the license key to use when setting up the New Relic agent" } +variable "new_relic_account_id" { + type = number + description = "New Relic Account ID" +} + +variable "new_relic_api_key" { + type = string + description = "New Relic API key" +} + variable "pgrst_jwt_secret" { type = string description = "the JWT signing secret for validating JWT tokens from api.data.gov" diff --git a/terraform/production/production.tf b/terraform/production/production.tf index ddc5005f82..187b50e0c6 100644 --- a/terraform/production/production.tf +++ b/terraform/production/production.tf @@ -2,6 +2,8 @@ module "production" { source = "../shared/modules/env" cf_space_name = "production" new_relic_license_key = var.new_relic_license_key + new_relic_account_id = var.new_relic_account_id + new_relic_api_key = var.new_relic_api_key pgrst_jwt_secret = var.pgrst_jwt_secret clamav_instances = 1 clamav_fs_instances = 1 diff --git a/terraform/production/variables.tf b/terraform/production/variables.tf index fe8367f7e1..5dc957c792 100644 --- a/terraform/production/variables.tf +++ b/terraform/production/variables.tf @@ -3,6 +3,16 @@ variable "new_relic_license_key" { description = "the license key to use when setting up the New Relic agent" } +variable "new_relic_account_id" { + type = number + description = "New Relic Account ID" +} + +variable "new_relic_api_key" { + type = string + description = "New Relic API key" +} + variable "pgrst_jwt_secret" { type = string description = "the JWT signing secret for validating JWT tokens from api.data.gov" diff --git a/terraform/shared/modules/env/newrelic.tf b/terraform/shared/modules/env/newrelic.tf index 244dd0190d..d240ad1b30 100644 --- a/terraform/shared/modules/env/newrelic.tf +++ b/terraform/shared/modules/env/newrelic.tf @@ -7,3 +7,10 @@ resource "cloudfoundry_user_provided_service" "credentials" { } tags = ["newrelic-creds"] } + +module "newrelic" { + source = "../newrelic" + cf_space_name = var.cf_space_name + new_relic_account_id = var.new_relic_account_id + new_relic_api_key = var.new_relic_api_key +} diff --git a/terraform/shared/modules/env/variables.tf b/terraform/shared/modules/env/variables.tf index ea9f021960..dfb532a147 100644 --- a/terraform/shared/modules/env/variables.tf +++ b/terraform/shared/modules/env/variables.tf @@ -100,3 +100,13 @@ variable "json_params" { type = string description = "Optional parameters used for service instance (-c)" } + +variable "new_relic_account_id" { + type = number + description = "New Relic Account ID" +} + +variable "new_relic_api_key" { + type = string + description = "New Relic API key" +} \ No newline at end of file diff --git a/terraform/shared/modules/newrelic/.terraform.lock.hcl b/terraform/shared/modules/newrelic/.terraform.lock.hcl new file mode 100644 index 0000000000..cfc9fa0fc6 --- /dev/null +++ b/terraform/shared/modules/newrelic/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/newrelic/newrelic" { + version = "3.38.0" + hashes = [ + "h1:+mJSDuRi8R1Plvgkr6LtfFwKSCP98pazqNt3Of2thZE=", + "zh:00ddd23b71aea83012a85819c2c41cc7ea9b13be1396f19c00392fe49d5ca965", + "zh:074c2d073e6079587439f6eb27e01c160dd190993599f648950c1a521ea3fa20", + "zh:16f8e6a90add8d794e72c63d9e3c149d6fa2ddef1cb3b151cf154da3a125028b", + "zh:1de682c1b7a870062b5be854da47a297966d0fae4337419a1ca9fffcf962e34e", + "zh:1e82cadd0bfc0095549621a23836482121a58e8012c54855ae4c53db5f3c9944", + "zh:596aebcab7974b63edfc3184d5722e3c4b45b6584833af7b790812c66a50bc5e", + "zh:61ff902f8e71f78c2998ca7f63987172232179d1d53b519af2bc3b4b15b56ae8", + "zh:662c1adb1fa6b965f0ac46258a73fd63e9ab063627948cc87d262695c5b00040", + "zh:7137adadcbe737800d935508ed6c70bd1a659677ac0f6b6fd8019eabf08f9b51", + "zh:7b149d38eb10ad666c5b8c263be4dbf7fc525daf586517d73b44b287131dcc6a", + "zh:9fa238a991a36db7cb20bbd4e87fef79c20da84550755f089b3b74578444b54a", + "zh:a02c0eeeef0f42ebe139f896d071bd6089a21e9f06dd789b85922c148dd18f13", + "zh:bf0fb80fcaf5221d47e1c2322a5428f4923b57bdf02acb21dd46bd5344fc0081", + "zh:dfc75e1605e2f6543263f73c4484209fcec24dc1ba41639c982b7dee7e88cf52", + "zh:edf31463e4a4c58f704010567b6d7bcff1293f1364f31676a5e8f1de31afad7e", + "zh:f0b4965e1625090fffaebb7d3309d55d162bcb4a759d7f1487014165cab897f8", + "zh:f10dae6fe1e94c2bce411acbf34a1275fe911d638d41e109a81bae814646e373", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/terraform/shared/modules/newrelic/alerts.tf b/terraform/shared/modules/newrelic/alerts.tf new file mode 100644 index 0000000000..5916025760 --- /dev/null +++ b/terraform/shared/modules/newrelic/alerts.tf @@ -0,0 +1,115 @@ +data "newrelic_entity" "gsa-fac" { + name = "gsa-fac-${var.cf_space_name}" + type = "APPLICATION" + domain = "APM" +} + +resource "newrelic_alert_policy" "alert_policy" { + name = "${var.cf_space_name}-alert-policy" +} + +resource "newrelic_notification_destination" "email_destination" { + account_id = var.new_relic_account_id + name = "email_destination" + type = "EMAIL" + + property { + key = "email" + value = "daniel.swick@gsa.gov, matthew.jadud@gsa.gov, alexander.steel@gsa.gov" + } +} + +resource "newrelic_notification_channel" "email_channel" { + account_id = var.new_relic_account_id + name = "${var.cf_space_name}_email_notification_channel" + type = "EMAIL" + product = "IINT" + destination_id = newrelic_notification_destination.email_destination.id + + property { + key = "subject" + value = "{{issueTitle}}" + } +} + +resource "newrelic_workflow" "alert_workflow" { + name = "${var.cf_space_name}_alert_workflow" + + muting_rules_handling = "DONT_NOTIFY_FULLY_MUTED_ISSUES" + + issues_filter { + name = "filter" + type = "FILTER" + + predicate { + attribute = "labels.policyIds" + operator = "EXACTLY_MATCHES" + values = [newrelic_alert_policy.alert_policy.id] + } + } + + destination { + channel_id = newrelic_notification_channel.email_channel.id + } +} + +/* +Alert if a log entry indicates that the fac-file-scanner found an infected file +*/ +resource "newrelic_nrql_alert_condition" "infected_file_found" { + policy_id = newrelic_alert_policy.alert_policy.id + name = "Infected File Found!" + enabled = true + + violation_time_limit_seconds = 259200 + + nrql { + query = "SELECT count(*) FROM Log WHERE tags.space_name ='${var.cf_space_name}' and message LIKE '%ScanResult.INFECTED%'" + } + + critical { + operator = "above_or_equals" + threshold = 1 + threshold_duration = 300 + threshold_occurrences = "at_least_once" + } + fill_option = "static" + fill_value = 0 + aggregation_window = 60 + aggregation_method = "event_flow" + aggregation_delay = 120 +} + +/* +Alert if the percentage of transactions resulting in an error surpasses a fixed threshold +*/ +resource "newrelic_nrql_alert_condition" "error_transactions" { + account_id = var.new_relic_account_id + policy_id = newrelic_alert_policy.alert_policy.id + + name = "Error Transactions (%)" + type = "static" + + nrql { + query = "SELECT percentage(count(*), WHERE error is true) FROM Transaction" + } + + critical { + operator = "above" + threshold = 5 + threshold_duration = 300 + threshold_occurrences = "all" + } + + warning { + operator = "above" + threshold = 3 + threshold_duration = 300 + threshold_occurrences = "all" + } + + fill_option = "none" + aggregation_window = 60 + aggregation_method = "event_flow" + aggregation_delay = 120 +} \ No newline at end of file diff --git a/terraform/shared/modules/newrelic/dashboards.tf b/terraform/shared/modules/newrelic/dashboards.tf new file mode 100644 index 0000000000..3800db273f --- /dev/null +++ b/terraform/shared/modules/newrelic/dashboards.tf @@ -0,0 +1,53 @@ +resource "newrelic_one_dashboard" "search_dashboard" { + name = "Search Dashboard (${var.cf_space_name})" + permissions = "public_read_only" + + page { + name = "Search" + + widget_billboard { + title = "Searches Per Hour" + + row = 1 + column = 1 + width = 3 + height = 3 + + nrql_query { + query = "SELECT count(*) as 'Total', rate(count(*), 1 minute) as 'Per Minute' FROM Transaction where request.uri like '%/dissemination/search%' and request.method = 'POST' and appName = 'gsa-fac-${var.cf_space_name}' since 1 hours AGO COMPARE WITH 1 week ago" + } + } + + widget_line { + title = "Search Traffic" + + + row = 1 + column = 4 + width = 6 + height = 3 + + nrql_query { + query = "SELECT count(*) FROM Transaction where request.uri like '%/dissemination/search%' and request.method = 'POST' and appName = 'gsa-fac-${var.cf_space_name}' since 4 hours AGO COMPARE WITH 1 week ago TIMESERIES" + } + + legend_enabled = true + + } + + widget_line { + title = "Search Response Time" + + row = 2 + column = 1 + width = 6 + height = 3 + + nrql_query { + query = "FROM Metric SELECT average(newrelic.timeslice.value) WHERE appName = 'gsa-fac-${var.cf_space_name}' WITH METRIC_FORMAT 'Custom/search' TIMESERIES SINCE 1 day ago COMPARE WITH 1 week ago" + } + + legend_enabled = true + } + } +} \ No newline at end of file diff --git a/terraform/shared/modules/newrelic/providers.tf b/terraform/shared/modules/newrelic/providers.tf new file mode 100644 index 0000000000..becda95264 --- /dev/null +++ b/terraform/shared/modules/newrelic/providers.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + newrelic = { + source = "newrelic/newrelic" + } + } +} + +provider "newrelic" { + account_id = var.new_relic_account_id + api_key = var.new_relic_api_key + region = "US" +} + diff --git a/terraform/shared/modules/newrelic/variables.tf b/terraform/shared/modules/newrelic/variables.tf new file mode 100644 index 0000000000..9586222844 --- /dev/null +++ b/terraform/shared/modules/newrelic/variables.tf @@ -0,0 +1,14 @@ +variable "new_relic_account_id" { + type = number + description = "New Relic Account ID" +} + +variable "new_relic_api_key" { + type = string + description = "New Relic API key" +} + +variable "cf_space_name" { + type = string + description = "cloud.gov space name for New Relic" +} \ No newline at end of file diff --git a/terraform/staging/staging.tf b/terraform/staging/staging.tf index e4532d01d1..43c1d4026a 100644 --- a/terraform/staging/staging.tf +++ b/terraform/staging/staging.tf @@ -2,6 +2,8 @@ module "staging" { source = "../shared/modules/env" cf_space_name = "staging" new_relic_license_key = var.new_relic_license_key + new_relic_account_id = var.new_relic_account_id + new_relic_api_key = var.new_relic_api_key pgrst_jwt_secret = var.pgrst_jwt_secret database_plan = "medium-gp-psql" diff --git a/terraform/staging/variables.tf b/terraform/staging/variables.tf index fe8367f7e1..5dc957c792 100644 --- a/terraform/staging/variables.tf +++ b/terraform/staging/variables.tf @@ -3,6 +3,16 @@ variable "new_relic_license_key" { description = "the license key to use when setting up the New Relic agent" } +variable "new_relic_account_id" { + type = number + description = "New Relic Account ID" +} + +variable "new_relic_api_key" { + type = string + description = "New Relic API key" +} + variable "pgrst_jwt_secret" { type = string description = "the JWT signing secret for validating JWT tokens from api.data.gov"