From 85c41678ea76b9da6d15ca1c8ce9bc5464382c80 Mon Sep 17 00:00:00 2001 From: Ashley Kleynhans Date: Fri, 31 May 2024 09:29:18 +0200 Subject: [PATCH] Improved logging and bumped Python runtime to 3.12 --- README.md | 37 ++++++++++++++++--------------------- requirements.txt | 1 + webhook.py | 32 ++++++++++++++++---------------- 3 files changed, 33 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index fba5754..51c495c 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Alertmanager Webhook Receiver to send notifications to Discord, Telegram, and PagerDuty -[![Python Version: 3.9]( -https://img.shields.io/badge/Python%20application-v3.9-blue -)](https://www.python.org/downloads/release/python-3913/) +[![Python Version: 3.12]( +https://img.shields.io/badge/Python%20application-v3.12-blue +)](https://www.python.org/downloads/release/python-3123/) [![License: GPL 3.0]( https://img.shields.io/github/license/ashleykleynhans/alertmanager-webhook )](https://opensource.org/licenses/GPL-3.0) @@ -13,25 +13,20 @@ https://img.shields.io/github/license/ashleykleynhans/alertmanager-webhook ```bash brew install ngrok ``` -2. Ensure your System Python3 version is 3.9, but greater than 3.9.1. +2. Ensure your System Python3 version is 3.12. ```bash python3 -V ``` -3. If your System Python is not 3.9: +3. If your System Python is not 3.12: ```bash -brew install python@3.9 -brew link python@3.9 +brew install python@3.12 +brew link python@3.12 ``` -4. If your System Python is 3.9 but not greater than 3.9.1: -```bash -brew update -brew upgrade python@3.9 -``` -5. [Create a new Discord App](https://discord.com/developers/applications). -6. Create your Discord channel where you want to receive your +4. [Create a new Discord App](https://discord.com/developers/applications). +5. Create your Discord channel where you want to receive your Alertmanager notifications. -7. Configure Alertmanager to send notifications to that channel. -8. Create a configuration file called `config.yml` in the same directory +6. Configure Alertmanager to send notifications to that channel. +7. Create a configuration file called `config.yml` in the same directory as the webhook script that looks like this: ```yml --- @@ -113,7 +108,7 @@ alertmanager_receivers: - url: "https://1d602d00.execute-api.us-east-1.amazonaws.com/alertmanager/warning" send_resolved: true ``` -### Promtheus Rules +### Prometheus Rules ```yaml groups: - name: haproxy.rules @@ -152,10 +147,10 @@ while ngrok is running **(be sure to use the https one)**. ## Deploy to AWS Lambda -1. Create a Python 3.9 Virtual Environment: +1. Create a Python 3.12 Virtual Environment: ```bash -python3 -m venv venv/py3.9 -source venv/py3.9/bin/activate +python3 -m venv venv/py3.12 +source venv/py3.12/bin/activate ``` 2. Upgrade pip. ```bash @@ -175,7 +170,7 @@ to configure your AWS Lambda deployment: "lambda_description": "Webhook to handle Alertmanager notifications", "profile_name": "default", "project_name": "alertmanager-webhook", - "runtime": "python3.9", + "runtime": "python3.12", "s3_bucket": "alertmanager-webhooks", "tags": { "service": "alertmanager-webhook" diff --git a/requirements.txt b/requirements.txt index 9ffd931..f24580c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,6 @@ boto3 zappa awsudo awscli +setuptools urllib3==1.26.18 requests==2.32.0 diff --git a/webhook.py b/webhook.py index 549e8a4..da1d5c3 100755 --- a/webhook.py +++ b/webhook.py @@ -261,10 +261,12 @@ def discord_handler(severity): title, description, hostname, status, application = parse_alert(alert, notification_system) if title is None and description is None: + logging.info('[DISCORD]: No title or description, no notification will be sent') continue # environment label must be present if 'environment' not in alert['labels']: + logging.info('[DISCORD]: No environment label, no notification will be sent') continue environment = alert['labels']['environment'] @@ -324,7 +326,7 @@ def discord_handler(severity): if response.status_code == 429: retry_after = discord_response['retry_after'] - logging.info(f'Discord rate limiting in place, retrying after: {retry_after}') + logging.warning(f'[DISCORD]: API rate limiting in place, retrying after: {retry_after}') time.sleep(retry_after) response = requests.post( @@ -340,7 +342,7 @@ def discord_handler(severity): discord_response = response.json() elif response.status_code != 200: - logging.info(f'Discord returned status code: {response.status_code}') + logging.error(f'[DISCORD]: API returned status code: {response.status_code}') responses.append(discord_response) @@ -367,10 +369,12 @@ def telegram_handler(severity): title, description, hostname, status, application = parse_alert(alert, notification_system) if title is None and description is None: + logging.info('[TELEGRAM]: No title or description, no notification will be sent') continue # environment label must be present if 'environment' not in alert['labels']: + logging.info('[TELEGRAM]: No environment label, no notification will be sent') continue environment = alert['labels']['environment'] @@ -407,7 +411,7 @@ def telegram_handler(severity): if response.status_code == 429: retry_after = telegram_response['retry_after'] - logging.info(f'Telegram rate limiting in place, retrying after: {retry_after}') + logging.warning(f'[TELEGRAM]: API rate limiting in place, retrying after: {retry_after}') time.sleep(retry_after) response = requests.post( @@ -421,7 +425,7 @@ def telegram_handler(severity): telegram_response = response.json() elif response.status_code != 200: - logging.info(f'Telegram returned status code: {response.status_code}') + logging.error(f'[TELEGRAM]: API returned status code: {response.status_code}') responses.append(telegram_response) @@ -431,7 +435,6 @@ def telegram_handler(severity): def pagerduty_handler(severity): notification_system = 'pagerduty' responses = [] - logging.debug(f'[PAGERDUTY]: Severity: {severity}') if severity != 'critical': return responses @@ -445,28 +448,26 @@ def pagerduty_handler(severity): ), 404) payload = request.get_json() - logging.debug('[PAGERDUTY]: Payload: ' + json.dumps(payload, indent=4, default=str)) url = 'https://events.pagerduty.com/v2/enqueue' for alert in payload['alerts']: title, description, hostname, status, application = parse_alert(alert, notification_system) - logging.debug(f'[PAGERDUTY]: Title: {title}') - logging.debug(f'[PAGERDUTY]: Description: {description}') - logging.debug(f'[PAGERDUTY]: Hostname: {hostname}') - logging.debug(f'[PAGERDUTY]: Status: {status}') - if title is None and description is None: + logging.info('[PAGERDUTY]: No title or description, no notification will be sent') continue # environment label must be present if 'environment' not in alert['labels']: + logging.info('[PAGERDUTY]: No environment label, no notification will be sent') continue # Only continue if there is a new alert that is firing if status == 'firing': + logging.info('[PAGERDUTY]: Status is firing, incident will be triggered') event_action = 'trigger' else: + logging.info('[PAGERDUTY]: Status is not firing, no incident will be triggered') continue environment = alert['labels']['environment'] @@ -506,9 +507,7 @@ def pagerduty_handler(severity): message += description logging.debug(f'[PAGERDUTY]: Service: {service}') - logging.debug(f'[PAGERDUTY]: Message: {message}') logging.debug(f'[PAGERDUTY]: Routing Key: {routing_key}') - logging.debug(f'[PAGERDUTY]: Event Action: {event_action}') payload = { 'payload': { @@ -529,7 +528,7 @@ def pagerduty_handler(severity): if response.status_code == 429: retry_after = pagerduty_response['retry_after'] - logging.info(f'PagerDuty rate limiting in place, retrying after: {retry_after}') + logging.warning(f'[PAGERDUTY]: API rate limiting in place, retrying after: {retry_after}') time.sleep(retry_after) response = requests.post( @@ -538,8 +537,9 @@ def pagerduty_handler(severity): ) pagerduty_response = response.json() - elif response.status_code != 200: - logging.info(f'PagerDuty returned status code: {response.status_code}') + # PagerDuty usually returns HTTP status code 202 + elif response.status_code != 200 and response.status_code != 202: + logging.error(f'[PAGERDUTY]: API returned status code: {response.status_code}') responses.append(pagerduty_response)