Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[#IOPID-2503] add alerts for download/delete failures #1334

Merged
merged 3 commits into from
Nov 25, 2024
+98 −0
Merged

[#IOPID-2503] add alerts for download/delete failures #1334

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e99b0d7
[#IOPID-2305] add alerts for download/delete failures
arcogabbo Nov 25, 2024
8371ddc
addressed review
arcogabbo Nov 25, 2024
1636a27
fix: removed isReplay filter
arcogabbo Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/domains/functions/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@
| [azurerm_monitor_metric_alert.function_assets_health_check](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_metric_alert) | resource |
| [azurerm_monitor_metric_alert.function_assets_http_server_errors](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_metric_alert) | resource |
| [azurerm_monitor_metric_alert.function_assets_response_time](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_metric_alert) | resource |
| [azurerm_monitor_scheduled_query_rules_alert_v2.alert_failed_delete_procedure](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert_v2) | resource |
| [azurerm_monitor_scheduled_query_rules_alert_v2.alert_failed_download_procedure](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert_v2) | resource |
| [azurerm_resource_group.admin_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
| [azurerm_resource_group.services_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource |
| [azurerm_app_service.appservice_app_backendli](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/app_service) | data source |
Expand Down Expand Up @@ -74,6 +76,7 @@
| [azurerm_key_vault_secret.fn_services_webhook_channel_aks_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source |
| [azurerm_key_vault_secret.fn_services_webhook_channel_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source |
| [azurerm_monitor_action_group.error_action_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source |
| [azurerm_monitor_action_group.io_auth_error_action_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source |
| [azurerm_monitor_action_group.io_com_action_group](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source |
| [azurerm_private_dns_zone.privatelink_blob_core](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source |
| [azurerm_private_dns_zone.privatelink_queue_core](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source |
Expand Down
5 changes: 5 additions & 0 deletions src/domains/functions/data.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ data "azurerm_monitor_action_group" "io_com_action_group" {
resource_group_name = "io-p-itn-msgs-rg-01"
}

data "azurerm_monitor_action_group" "io_auth_error_action_group" {
name = "io-p-itn-auth-error-ag-01"
resource_group_name = "io-p-itn-auth-common-rg-01"
}

data "azurerm_private_dns_zone" "privatelink_queue_core" {
name = "privatelink.queue.core.windows.net"
resource_group_name = local.rg_common_name
Expand Down
90 changes: 90 additions & 0 deletions src/domains/functions/function_admin.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,96 @@ module "function_admin_staging_slot" {
tags = var.tags
}

// ----------------------------------------------------
// Alerts
// ----------------------------------------------------
resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_delete_procedure" {
enabled = true
name = "[IO-AUTH | ${module.function_admin.name}] Found one or more failed DELETE procedures"
resource_group_name = azurerm_resource_group.admin_rg.name
scopes = [data.azurerm_application_insights.application_insights.id]
description = <<EOT
Found one or more failed DELETE procedures.
Check ${local.function_admin.app_settings_common.FAILED_USER_DATA_PROCESSING_TABLE} table in
${data.azurerm_storage_account.storage_api.name} storage account for more details
EOT

severity = 1
auto_mitigation_enabled = false
location = azurerm_resource_group.admin_rg.location

// check once every minute(evaluation_frequency)
// on the last minute of data(window_duration)
evaluation_frequency = "P1D"
window_duration = "P1D"
arcogabbo marked this conversation as resolved.
Show resolved Hide resolved

criteria {
query = <<-QUERY
exceptions
arcogabbo marked this conversation as resolved.
Show resolved Hide resolved
| where customDimensions.name startswith "user.data.delete"
| where customDimensions.isReplay == true
arcogabbo marked this conversation as resolved.
Show resolved Hide resolved
QUERY
operator = "GreaterThanOrEqual"
time_aggregation_method = "Count"
threshold = 1
failing_periods {
minimum_failing_periods_to_trigger_alert = 1
number_of_evaluation_periods = 1
}
}

# Action groups for alerts
action {
action_groups = [data.azurerm_monitor_action_group.io_auth_error_action_group.id]
}

tags = var.tags
}

resource "azurerm_monitor_scheduled_query_rules_alert_v2" "alert_failed_download_procedure" {
enabled = true
name = "[IO-AUTH | ${module.function_admin.name}] Found one or more failed DOWNLOAD procedures"
resource_group_name = azurerm_resource_group.admin_rg.name
scopes = [data.azurerm_application_insights.application_insights.id]
description = <<EOT
Found one or more failed DOWNLOAD procedures.
Check ${local.function_admin.app_settings_common.FAILED_USER_DATA_PROCESSING_TABLE} table in
${data.azurerm_storage_account.storage_api.name} storage account for more details
EOT

severity = 1
auto_mitigation_enabled = false
location = azurerm_resource_group.admin_rg.location

// check once every minute(evaluation_frequency)
// on the last minute of data(window_duration)
evaluation_frequency = "P1D"
window_duration = "P1D"

criteria {
query = <<-QUERY
exceptions
arcogabbo marked this conversation as resolved.
Show resolved Hide resolved
| where customDimensions.name startswith "user.data.download"
| where customDimensions.isReplay == true
arcogabbo marked this conversation as resolved.
Show resolved Hide resolved
QUERY
operator = "GreaterThanOrEqual"
time_aggregation_method = "Count"
threshold = 1
failing_periods {
minimum_failing_periods_to_trigger_alert = 1
number_of_evaluation_periods = 1
}
}

# Action groups for alerts
action {
action_groups = [data.azurerm_monitor_action_group.io_auth_error_action_group.id]
}

tags = var.tags
}
// -----------------------------------------------------

resource "azurerm_monitor_autoscale_setting" "function_admin" {
name = format("%s-autoscale", module.function_admin.name)
resource_group_name = azurerm_resource_group.admin_rg.name
Expand Down