[pxc-db] Add binlog-collector alerts

Add binlog-collector alerts based on the last successful event timestamp * GaleraClusterBinlogProcessingTooOld * GaleraClusterBinlogUploadTooOld
sapcc · Dec 23, 2024 · a2e21c3 · a2e21c3
1 parent a1a1a9a
commit a2e21c3
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 3 deletions.
diff --git a/common/pxc-db/Chart.yaml b/common/pxc-db/Chart.yaml
@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.2
+version: 0.2.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

diff --git a/common/pxc-db/templates/alerts/_backup.alerts.tpl b/common/pxc-db/templates/alerts/_backup.alerts.tpl
@@ -25,5 +25,35 @@
       playbook: ''
       support_group: {{ required ".Values.alerts.support_group missing" .Values.alerts.support_group }}
     annotations:
-      description: "{{ include "pxc-db.fullname" . }} cluster has no new backups completed earlier than 36 hours ago."
-      summary: "{{ include "pxc-db.fullname" . }} cluster has no new backups completed earlier than 36 hours ago."
+      description: "{{ include "pxc-db.fullname" . }} cluster has no new full backups completed earlier than 36 hours ago."
+      summary: "{{ include "pxc-db.fullname" . }} cluster has no new full backups completed earlier than 36 hours ago."
+
+{{- if .Values.backup.pitr.enabled }}
+  - alert: {{ include "pxc-db.alerts.service" . | camelcase }}GaleraClusterBinlogProcessingTooOld
+    expr: (time() - pxc_binlog_collector_last_processing_timestamp > 1800)
+    for: 30m
+    labels:
+      context: database
+      service: {{ include "pxc-db.alerts.service" . }}
+      severity: info
+      tier: {{ required ".Values.alerts.tier missing" .Values.alerts.tier }}
+      playbook: ''
+      support_group: {{ required ".Values.alerts.support_group missing" .Values.alerts.support_group }}
+    annotations:
+      description: "More than 30 minutes passed since the last cluster {{ include "pxc-db.fullname" . }} binlog processing."
+      summary: "{{ include "pxc-db.fullname" . }} cluster binlog processing is too old."
+
+  - alert: {{ include "pxc-db.alerts.service" . | camelcase }}GaleraClusterBinlogUploadTooOld
+    expr: (time() - pxc_binlog_collector_last_upload_timestamp > 1800)
+    for: 30m
+    labels:
+      context: database
+      service: {{ include "pxc-db.alerts.service" . }}
+      severity: info
+      tier: {{ required ".Values.alerts.tier missing" .Values.alerts.tier }}
+      playbook: ''
+      support_group: {{ required ".Values.alerts.support_group missing" .Values.alerts.support_group }}
+    annotations:
+      description: "More than 30 minutes passed since the last cluster {{ include "pxc-db.fullname" . }} binlog upload."
+      summary: "{{ include "pxc-db.fullname" . }} cluster binlog upload is too old."
+{{- end }}