Improved logging and bumped Python runtime to 3.12

ashleykleynhans · May 31, 2024 · 85c4167 · 85c4167
1 parent 382af86
commit 85c4167
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 37 deletions.
diff --git a/README.md b/README.md
@@ -1,8 +1,8 @@
 # Alertmanager Webhook Receiver to send notifications to Discord, Telegram, and PagerDuty
 
-[![Python Version: 3.9](
-https://img.shields.io/badge/Python%20application-v3.9-blue
-)](https://www.python.org/downloads/release/python-3913/)
+[![Python Version: 3.12](
+https://img.shields.io/badge/Python%20application-v3.12-blue
+)](https://www.python.org/downloads/release/python-3123/)
 [![License: GPL 3.0](
 https://img.shields.io/github/license/ashleykleynhans/alertmanager-webhook
 )](https://opensource.org/licenses/GPL-3.0)
@@ -13,25 +13,20 @@ https://img.shields.io/github/license/ashleykleynhans/alertmanager-webhook
 ```bash
 brew install ngrok
 ```
-2. Ensure your System Python3 version is 3.9, but greater than 3.9.1.
+2. Ensure your System Python3 version is 3.12.
 ```bash
 python3 -V
 ```
-3. If your System Python is not 3.9:
+3. If your System Python is not 3.12:
 ```bash
-brew install python@3.9
-brew link python@3.9
+brew install python@3.12
+brew link python@3.12
 ```
-4. If your System Python is 3.9 but not greater than 3.9.1:
-```bash
-brew update
-brew upgrade [email protected]
-```
-5. [Create a new Discord App](https://discord.com/developers/applications).
-6. Create your Discord channel where you want to receive your
+4. [Create a new Discord App](https://discord.com/developers/applications).
+5. Create your Discord channel where you want to receive your
    Alertmanager notifications.
-7. Configure Alertmanager to send notifications to that channel.
-8. Create a configuration file called `config.yml` in the same directory
+6. Configure Alertmanager to send notifications to that channel.
+7. Create a configuration file called `config.yml` in the same directory
    as the webhook script that looks like this:
 ```yml
 ---
@@ -113,7 +108,7 @@ alertmanager_receivers:
        - url: "https://1d602d00.execute-api.us-east-1.amazonaws.com/alertmanager/warning"
          send_resolved: true
 ```
-### Promtheus Rules
+### Prometheus Rules
 ```yaml
 groups:
   - name: haproxy.rules
@@ -152,10 +147,10 @@ while ngrok is running **(be sure to use the https one)**.
 
 ## Deploy to AWS Lambda
 
-1. Create a Python 3.9 Virtual Environment:
+1. Create a Python 3.12 Virtual Environment:
 ```bash
-python3 -m venv venv/py3.9
-source venv/py3.9/bin/activate
+python3 -m venv venv/py3.12
+source venv/py3.12/bin/activate
 ```
 2. Upgrade pip.
 ```bash
@@ -175,7 +170,7 @@ to configure your AWS Lambda deployment:
         "lambda_description": "Webhook to handle Alertmanager notifications",
         "profile_name": "default",
         "project_name": "alertmanager-webhook",
-        "runtime": "python3.9",
+        "runtime": "python3.12",
         "s3_bucket": "alertmanager-webhooks",
         "tags": {
             "service": "alertmanager-webhook"

diff --git a/requirements.txt b/requirements.txt
@@ -3,5 +3,6 @@ boto3
 zappa
 awsudo
 awscli
+setuptools
 urllib3==1.26.18
 requests==2.32.0
diff --git a/webhook.py b/webhook.py
@@ -261,10 +261,12 @@ def discord_handler(severity):
         title, description, hostname, status, application = parse_alert(alert, notification_system)
 
         if title is None and description is None:
+            logging.info('[DISCORD]: No title or description, no notification will be sent')
             continue
 
         # environment label must be present
         if 'environment' not in alert['labels']:
+            logging.info('[DISCORD]: No environment label, no notification will be sent')
             continue
 
         environment = alert['labels']['environment']
@@ -324,7 +326,7 @@ def discord_handler(severity):
 
         if response.status_code == 429:
             retry_after = discord_response['retry_after']
-            logging.info(f'Discord rate limiting in place, retrying after: {retry_after}')
+            logging.warning(f'[DISCORD]: API rate limiting in place, retrying after: {retry_after}')
             time.sleep(retry_after)
 
             response = requests.post(
@@ -340,7 +342,7 @@ def discord_handler(severity):
 
             discord_response = response.json()
         elif response.status_code != 200:
-            logging.info(f'Discord returned status code: {response.status_code}')
+            logging.error(f'[DISCORD]: API returned status code: {response.status_code}')
 
         responses.append(discord_response)
 
@@ -367,10 +369,12 @@ def telegram_handler(severity):
         title, description, hostname, status, application = parse_alert(alert, notification_system)
 
         if title is None and description is None:
+            logging.info('[TELEGRAM]: No title or description, no notification will be sent')
             continue
 
         # environment label must be present
         if 'environment' not in alert['labels']:
+            logging.info('[TELEGRAM]: No environment label, no notification will be sent')
             continue
 
         environment = alert['labels']['environment']
@@ -407,7 +411,7 @@ def telegram_handler(severity):
 
         if response.status_code == 429:
             retry_after = telegram_response['retry_after']
-            logging.info(f'Telegram rate limiting in place, retrying after: {retry_after}')
+            logging.warning(f'[TELEGRAM]: API rate limiting in place, retrying after: {retry_after}')
             time.sleep(retry_after)
 
             response = requests.post(
@@ -421,7 +425,7 @@ def telegram_handler(severity):
 
             telegram_response = response.json()
         elif response.status_code != 200:
-            logging.info(f'Telegram returned status code: {response.status_code}')
+            logging.error(f'[TELEGRAM]: API returned status code: {response.status_code}')
 
         responses.append(telegram_response)
 
@@ -431,7 +435,6 @@ def telegram_handler(severity):
 def pagerduty_handler(severity):
     notification_system = 'pagerduty'
     responses = []
-    logging.debug(f'[PAGERDUTY]: Severity: {severity}')
 
     if severity != 'critical':
         return responses
@@ -445,28 +448,26 @@ def pagerduty_handler(severity):
         ), 404)
 
     payload = request.get_json()
-    logging.debug('[PAGERDUTY]: Payload: ' + json.dumps(payload, indent=4, default=str))
     url = 'https://events.pagerduty.com/v2/enqueue'
 
     for alert in payload['alerts']:
         title, description, hostname, status, application = parse_alert(alert, notification_system)
 
-        logging.debug(f'[PAGERDUTY]: Title: {title}')
-        logging.debug(f'[PAGERDUTY]: Description: {description}')
-        logging.debug(f'[PAGERDUTY]: Hostname: {hostname}')
-        logging.debug(f'[PAGERDUTY]: Status: {status}')
-
         if title is None and description is None:
+            logging.info('[PAGERDUTY]: No title or description, no notification will be sent')
             continue
 
         # environment label must be present
         if 'environment' not in alert['labels']:
+            logging.info('[PAGERDUTY]: No environment label, no notification will be sent')
             continue
 
         # Only continue if there is a new alert that is firing
         if status == 'firing':
+            logging.info('[PAGERDUTY]: Status is firing, incident will be triggered')
             event_action = 'trigger'
         else:
+            logging.info('[PAGERDUTY]: Status is not firing, no incident will be triggered')
             continue
 
         environment = alert['labels']['environment']
@@ -506,9 +507,7 @@ def pagerduty_handler(severity):
         message += description
 
         logging.debug(f'[PAGERDUTY]: Service: {service}')
-        logging.debug(f'[PAGERDUTY]: Message: {message}')
         logging.debug(f'[PAGERDUTY]: Routing Key: {routing_key}')
-        logging.debug(f'[PAGERDUTY]: Event Action: {event_action}')
 
         payload = {
             'payload': {
@@ -529,7 +528,7 @@ def pagerduty_handler(severity):
 
         if response.status_code == 429:
             retry_after = pagerduty_response['retry_after']
-            logging.info(f'PagerDuty rate limiting in place, retrying after: {retry_after}')
+            logging.warning(f'[PAGERDUTY]: API rate limiting in place, retrying after: {retry_after}')
             time.sleep(retry_after)
 
             response = requests.post(
@@ -538,8 +537,9 @@ def pagerduty_handler(severity):
             )
 
             pagerduty_response = response.json()
-        elif response.status_code != 200:
-            logging.info(f'PagerDuty returned status code: {response.status_code}')
+        # PagerDuty usually returns HTTP status code 202
+        elif response.status_code != 200 and response.status_code != 202:
+            logging.error(f'[PAGERDUTY]: API returned status code: {response.status_code}')
 
         responses.append(pagerduty_response)