From e99b0d79706c343f8235d5cbf8648c00f0fc0e4d Mon Sep 17 00:00:00 2001 From: arcogabbo Date: Mon, 25 Nov 2024 14:51:59 +0100 Subject: [PATCH 1/3] [#IOPID-2305] add alerts for download/delete failures --- src/domains/functions/README.md | 3 + src/domains/functions/data.tf | 5 ++ src/domains/functions/function_admin.tf | 90 +++++++++++++++++++++++++ 3 files changed, 98 insertions(+) diff --git a/src/domains/functions/README.md b/src/domains/functions/README.md index b28a3a8af..a62ef00a8 100644 --- a/src/domains/functions/README.md +++ b/src/domains/functions/README.md @@ -42,6 +42,8 @@ | [azurerm_monitor_metric_alert.function_assets_health_check](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_metric_alert) | resource | | [azurerm_monitor_metric_alert.function_assets_http_server_errors](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_metric_alert) | resource | | [azurerm_monitor_metric_alert.function_assets_response_time](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_metric_alert) | resource | +| [azurerm_monitor_scheduled_query_rules_alert_v2.alert_failed_delete_procedure](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert_v2) | resource | +| [azurerm_monitor_scheduled_query_rules_alert_v2.alert_failed_download_procedure](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert_v2) | resource | | [azurerm_resource_group.admin_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.services_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_app_service.appservice_app_backendli](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/app_service) | data source | @@ -74,6 +76,7 @@ | [azurerm_key_vault_secret.fn_services_webhook_channel_aks_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.fn_services_webhook_channel_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_monitor_action_group.error_action_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | +| [azurerm_monitor_action_group.io_auth_error_action_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_monitor_action_group.io_com_action_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_private_dns_zone.privatelink_blob_core](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | | [azurerm_private_dns_zone.privatelink_queue_core](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | diff --git a/src/domains/functions/data.tf b/src/domains/functions/data.tf index c076eb802..c554850e9 100644 --- a/src/domains/functions/data.tf +++ b/src/domains/functions/data.tf @@ -33,6 +33,11 @@ data "azurerm_monitor_action_group" "io_com_action_group" { resource_group_name = "io-p-itn-msgs-rg-01" } +data "azurerm_monitor_action_group" "io_auth_error_action_group" { + name = "io-p-itn-auth-error-ag-01" + resource_group_name = "io-p-itn-auth-common-rg-01" +} + data "azurerm_private_dns_zone" "privatelink_queue_core" { name = "privatelink.queue.core.windows.net" resource_group_name = local.rg_common_name diff --git a/src/domains/functions/function_admin.tf b/src/domains/functions/function_admin.tf index 04b3d7228..80bae3d5c 100644 --- a/src/domains/functions/function_admin.tf +++ b/src/domains/functions/function_admin.tf @@ -299,6 +299,96 @@ module "function_admin_staging_slot" { tags = var.tags } +// ---------------------------------------------------- +// Alerts +// ---------------------------------------------------- +resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_delete_procedure" { + enabled = true + name = "[IO-AUTH | ${module.function_admin.name}] Found one or more failed DELETE procedures" + resource_group_name = azurerm_resource_group.admin_rg.name + scopes = [data.azurerm_application_insights.application_insights.id] + description = < Date: Mon, 25 Nov 2024 16:10:07 +0100 Subject: [PATCH 2/3] addressed review --- src/domains/functions/function_admin.tf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/domains/functions/function_admin.tf b/src/domains/functions/function_admin.tf index 80bae3d5c..34d9dbe06 100644 --- a/src/domains/functions/function_admin.tf +++ b/src/domains/functions/function_admin.tf @@ -317,14 +317,15 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_delete_p auto_mitigation_enabled = false location = azurerm_resource_group.admin_rg.location - // check once every minute(evaluation_frequency) - // on the last minute of data(window_duration) + // check once every day(evaluation_frequency) + // on the last 24 hours of data(window_duration) evaluation_frequency = "P1D" window_duration = "P1D" criteria { query = <<-QUERY exceptions +| where cloud_RoleName == "${module.function_admin.name}" | where customDimensions.name startswith "user.data.delete" | where customDimensions.isReplay == true QUERY @@ -360,14 +361,15 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_download auto_mitigation_enabled = false location = azurerm_resource_group.admin_rg.location - // check once every minute(evaluation_frequency) - // on the last minute of data(window_duration) + // check once every day(evaluation_frequency) + // on the last 24 hours of data(window_duration) evaluation_frequency = "P1D" window_duration = "P1D" criteria { query = <<-QUERY exceptions +| where cloud_RoleName == "${module.function_admin.name}" | where customDimensions.name startswith "user.data.download" | where customDimensions.isReplay == true QUERY From 1636a27919134d045b53b406175c4f0c4adff880 Mon Sep 17 00:00:00 2001 From: arcogabbo Date: Mon, 25 Nov 2024 16:55:25 +0100 Subject: [PATCH 3/3] fix: removed isReplay filter --- src/domains/functions/function_admin.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/domains/functions/function_admin.tf b/src/domains/functions/function_admin.tf index 34d9dbe06..8335e4350 100644 --- a/src/domains/functions/function_admin.tf +++ b/src/domains/functions/function_admin.tf @@ -327,7 +327,6 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_delete_p exceptions | where cloud_RoleName == "${module.function_admin.name}" | where customDimensions.name startswith "user.data.delete" -| where customDimensions.isReplay == true QUERY operator = "GreaterThanOrEqual" time_aggregation_method = "Count" @@ -371,7 +370,6 @@ resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_download exceptions | where cloud_RoleName == "${module.function_admin.name}" | where customDimensions.name startswith "user.data.download" -| where customDimensions.isReplay == true QUERY operator = "GreaterThanOrEqual" time_aggregation_method = "Count"