Configure New Relic from Terraform (#4005)

* newrelic module

* set nr tf vars

* define nr vars in env

* define nr vars in preview module

* define nr vars in preview module

* try a list of emails

* emails as separate properties

* comma separated

* adjust configuration

* add nr vars to other envs

* add comments for alert conditions

* tf formatting

* tf formatting

* tf formatting

* tf formatting

* add tf vars to plan workflow

.github/workflows/terraform-apply-env.yml

@@ -22,6 +22,8 @@ jobs:
       TF_VAR_cf_user: ${{ secrets.CF_USERNAME }}
       TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }}
       TF_VAR_new_relic_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+      TF_VAR_new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
+      TF_VAR_new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
       TF_VAR_pgrst_jwt_secret: ${{ secrets.PGRST_JWT_SECRET }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       TERRAFORM_PRE_RUN: |

.github/workflows/terraform-plan-env.yml

@@ -17,6 +17,8 @@ jobs:
       TF_VAR_cf_user: ${{ secrets.CF_USERNAME }}
       TF_VAR_cf_password: ${{ secrets.CF_PASSWORD }}
       TF_VAR_new_relic_license_key: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+      TF_VAR_new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
+      TF_VAR_new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
       TF_VAR_pgrst_jwt_secret: ${{ secrets.PGRST_JWT_SECRET }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       TERRAFORM_PRE_RUN: |

terraform/dev/dev.tf

@@ -2,6 +2,8 @@ module "dev" {
   source                = "../shared/modules/env"
   cf_space_name         = "dev"
   new_relic_license_key = var.new_relic_license_key
+  new_relic_account_id  = var.new_relic_account_id
+  new_relic_api_key     = var.new_relic_api_key
   pgrst_jwt_secret      = var.pgrst_jwt_secret
 
   database_plan         = "medium-gp-psql"

terraform/dev/variables.tf

@@ -3,6 +3,16 @@ variable "new_relic_license_key" {
   description = "the license key to use when setting up the New Relic agent"
 }
 
+variable "new_relic_account_id" {
+  type        = number
+  description = "New Relic Account ID"
+}
+
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API key"
+}
+
 variable "pgrst_jwt_secret" {
   type        = string
   description = "the JWT signing secret for validating JWT tokens from api.data.gov"

terraform/preview/preview.tf

@@ -2,6 +2,8 @@ module "preview" {
   source                = "../shared/modules/env"
   cf_space_name         = "preview"
   new_relic_license_key = var.new_relic_license_key
+  new_relic_account_id  = var.new_relic_account_id
+  new_relic_api_key     = var.new_relic_api_key
   pgrst_jwt_secret      = var.pgrst_jwt_secret
 
   database_plan         = "medium-gp-psql"

terraform/preview/variables.tf

@@ -3,6 +3,16 @@ variable "new_relic_license_key" {
   description = "the license key to use when setting up the New Relic agent"
 }
 
+variable "new_relic_account_id" {
+  type        = number
+  description = "New Relic Account ID"
+}
+
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API key"
+}
+
 variable "pgrst_jwt_secret" {
   type        = string
   description = "the JWT signing secret for validating JWT tokens from api.data.gov"

terraform/production/production.tf

@@ -2,6 +2,8 @@ module "production" {
   source                = "../shared/modules/env"
   cf_space_name         = "production"
   new_relic_license_key = var.new_relic_license_key
+  new_relic_account_id  = var.new_relic_account_id
+  new_relic_api_key     = var.new_relic_api_key
   pgrst_jwt_secret      = var.pgrst_jwt_secret
   clamav_instances      = 1
   clamav_fs_instances   = 1

terraform/production/variables.tf

@@ -3,6 +3,16 @@ variable "new_relic_license_key" {
   description = "the license key to use when setting up the New Relic agent"
 }
 
+variable "new_relic_account_id" {
+  type        = number
+  description = "New Relic Account ID"
+}
+
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API key"
+}
+
 variable "pgrst_jwt_secret" {
   type        = string
   description = "the JWT signing secret for validating JWT tokens from api.data.gov"

terraform/shared/modules/env/newrelic.tf

@@ -7,3 +7,10 @@ resource "cloudfoundry_user_provided_service" "credentials" {
   }
   tags = ["newrelic-creds"]
 }
+
+module "newrelic" {
+  source               = "../newrelic"
+  cf_space_name        = var.cf_space_name
+  new_relic_account_id = var.new_relic_account_id
+  new_relic_api_key    = var.new_relic_api_key
+}

terraform/shared/modules/env/variables.tf

@@ -100,3 +100,13 @@ variable "json_params" {
   type        = string
   description = "Optional parameters used for service instance (-c)"
 }
+
+variable "new_relic_account_id" {
+  type        = number
+  description = "New Relic Account ID"
+}
+
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API key"
+}

terraform/shared/modules/newrelic/alerts.tf

@@ -0,0 +1,115 @@
+data "newrelic_entity" "gsa-fac" {
+  name   = "gsa-fac-${var.cf_space_name}"
+  type   = "APPLICATION"
+  domain = "APM"
+}
+
+resource "newrelic_alert_policy" "alert_policy" {
+  name = "${var.cf_space_name}-alert-policy"
+}
+
+resource "newrelic_notification_destination" "email_destination" {
+  account_id = var.new_relic_account_id
+  name       = "email_destination"
+  type       = "EMAIL"
+
+  property {
+    key   = "email"
+    value = "[email protected], [email protected], [email protected]"
+  }
+}
+
+resource "newrelic_notification_channel" "email_channel" {
+  account_id     = var.new_relic_account_id
+  name           = "${var.cf_space_name}_email_notification_channel"
+  type           = "EMAIL"
+  product        = "IINT"
+  destination_id = newrelic_notification_destination.email_destination.id
+
+  property {
+    key   = "subject"
+    value = "{{issueTitle}}"
+  }
+}
+
+resource "newrelic_workflow" "alert_workflow" {
+  name = "${var.cf_space_name}_alert_workflow"
+
+  muting_rules_handling = "DONT_NOTIFY_FULLY_MUTED_ISSUES"
+
+  issues_filter {
+    name = "filter"
+    type = "FILTER"
+
+    predicate {
+      attribute = "labels.policyIds"
+      operator  = "EXACTLY_MATCHES"
+      values    = [newrelic_alert_policy.alert_policy.id]
+    }
+  }
+
+  destination {
+    channel_id = newrelic_notification_channel.email_channel.id
+  }
+}
+
+/*
+Alert if a log entry indicates that the fac-file-scanner found an infected file
+*/
+resource "newrelic_nrql_alert_condition" "infected_file_found" {
+  policy_id = newrelic_alert_policy.alert_policy.id
+  name      = "Infected File Found!"
+  enabled   = true
+
+  violation_time_limit_seconds = 259200
+
+  nrql {
+    query = "SELECT count(*) FROM Log WHERE tags.space_name ='${var.cf_space_name}' and message LIKE '%ScanResult.INFECTED%'"
+  }
+
+  critical {
+    operator              = "above_or_equals"
+    threshold             = 1
+    threshold_duration    = 300
+    threshold_occurrences = "at_least_once"
+  }
+  fill_option        = "static"
+  fill_value         = 0
+  aggregation_window = 60
+  aggregation_method = "event_flow"
+  aggregation_delay  = 120
+}
+
+/*
+Alert if the percentage of transactions resulting in an error surpasses a fixed threshold
+*/
+resource "newrelic_nrql_alert_condition" "error_transactions" {
+  account_id = var.new_relic_account_id
+  policy_id  = newrelic_alert_policy.alert_policy.id
+
+  name = "Error Transactions (%)"
+  type = "static"
+
+  nrql {
+    query = "SELECT percentage(count(*), WHERE error is true) FROM Transaction"
+  }
+
+  critical {
+    operator              = "above"
+    threshold             = 5
+    threshold_duration    = 300
+    threshold_occurrences = "all"
+  }
+
+  warning {
+    operator              = "above"
+    threshold             = 3
+    threshold_duration    = 300
+    threshold_occurrences = "all"
+  }
+
+  fill_option        = "none"
+  aggregation_window = 60
+  aggregation_method = "event_flow"
+  aggregation_delay  = 120
+}

terraform/shared/modules/newrelic/dashboards.tf

@@ -0,0 +1,53 @@
+resource "newrelic_one_dashboard" "search_dashboard" {
+  name        = "Search Dashboard (${var.cf_space_name})"
+  permissions = "public_read_only"
+
+  page {
+    name = "Search"
+
+    widget_billboard {
+      title = "Searches Per Hour"
+
+      row    = 1
+      column = 1
+      width  = 3
+      height = 3
+
+      nrql_query {
+        query = "SELECT count(*) as 'Total', rate(count(*), 1 minute) as 'Per Minute' FROM Transaction where request.uri like '%/dissemination/search%' and request.method = 'POST' and appName = 'gsa-fac-${var.cf_space_name}' since 1 hours AGO COMPARE WITH 1 week ago"
+      }
+    }
+
+    widget_line {
+      title = "Search Traffic"
+
+
+      row    = 1
+      column = 4
+      width  = 6
+      height = 3
+
+      nrql_query {
+        query = "SELECT count(*) FROM Transaction where request.uri like '%/dissemination/search%' and request.method = 'POST' and appName = 'gsa-fac-${var.cf_space_name}' since 4 hours AGO COMPARE WITH 1 week ago TIMESERIES"
+      }
+
+      legend_enabled = true
+
+    }
+
+    widget_line {
+      title = "Search Response Time"
+
+      row    = 2
+      column = 1
+      width  = 6
+      height = 3
+
+      nrql_query {
+        query = "FROM Metric SELECT average(newrelic.timeslice.value) WHERE appName = 'gsa-fac-${var.cf_space_name}' WITH METRIC_FORMAT 'Custom/search' TIMESERIES SINCE 1 day ago COMPARE WITH 1 week ago"
+      }
+
+      legend_enabled = true
+    }
+  }
+}

terraform/shared/modules/newrelic/providers.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    newrelic = {
+      source = "newrelic/newrelic"
+    }
+  }
+}
+
+provider "newrelic" {
+  account_id = var.new_relic_account_id
+  api_key    = var.new_relic_api_key
+  region     = "US"
+}
+

terraform/shared/modules/newrelic/variables.tf

@@ -0,0 +1,14 @@
+variable "new_relic_account_id" {
+  type        = number
+  description = "New Relic Account ID"
+}
+
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API key"
+}
+
+variable "cf_space_name" {
+  type        = string
+  description = "cloud.gov space name for New Relic"
+}

terraform/staging/staging.tf

@@ -2,6 +2,8 @@ module "staging" {
   source                = "../shared/modules/env"
   cf_space_name         = "staging"
   new_relic_license_key = var.new_relic_license_key
+  new_relic_account_id  = var.new_relic_account_id
+  new_relic_api_key     = var.new_relic_api_key
   pgrst_jwt_secret      = var.pgrst_jwt_secret
 
   database_plan         = "medium-gp-psql"

terraform/staging/variables.tf

@@ -3,6 +3,16 @@ variable "new_relic_license_key" {
   description = "the license key to use when setting up the New Relic agent"
 }
 
+variable "new_relic_account_id" {
+  type        = number
+  description = "New Relic Account ID"
+}
+
+variable "new_relic_api_key" {
+  type        = string
+  description = "New Relic API key"
+}
+
 variable "pgrst_jwt_secret" {
   type        = string
   description = "the JWT signing secret for validating JWT tokens from api.data.gov"