Merge pull request #118 from vshn/add/prometheusrule

Add Prometheus rule for storage
vshn · Mar 21, 2023 · a1a6410 · a1a6410
2 parents 7f70c77 + e822ba7
commit a1a6410
Show file tree

Hide file tree

Showing 8 changed files with 273 additions and 0 deletions.
diff --git a/component/provider.jsonnet b/component/provider.jsonnet
@@ -168,6 +168,11 @@ local controllerConfigRef(config) =
           resources: [ 'vshnpostgresqls' ],
           verbs: [ 'get' ],
         },
+        {
+          apiGroups: [ 'monitoring.coreos.com' ],
+          resources: [ 'prometheusrules' ],
+          verbs: [ 'get', 'list', 'watch', 'update', 'patch', 'create', 'delete' ],
+        },
       ],
     };
     local rolebinding = kube.ClusterRoleBinding('crossplane:provider:provider-kubernetes:system:custom') {

diff --git a/component/vshn_postgres.jsonnet b/component/vshn_postgres.jsonnet
@@ -842,6 +842,59 @@ local maintenanceJob = {
   ] + convertToCron(),
 };
 
+local prometheusRule = {
+  base: comp.KubeObject('monitoring.coreos.com/v1', 'PrometheusRule') + {
+    spec+: {
+      forProvider+: {
+        manifest+: {
+          metadata: {
+            name: 'postgresql-storage-rules',
+          },
+          spec: {
+            groups: [
+              {
+                name: 'postgresql-storage',
+                rules: [
+                  {
+                    alert: 'PostgreSQLPersistentVolumeFillingUp',
+                    annotations: {
+                      description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim\n              }} in Namespace {{ $labels.namespace }} is only {{ $value |\n              humanizePercentage }} free.',
+                      runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup',
+                      summary: 'PersistentVolume is filling up.',
+                    },
+                    expr: '(\n            kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}\n              /\n            kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}\n          ) < 0.03\n          and\n          kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0\n          unless on(namespace, persistentvolumeclaim)\n          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1\n          unless on(namespace, persistentvolumeclaim)\n          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1',
+                    'for': '1m',
+                    labels: {
+                      severity: 'critical',
+                    },
+                  },
+                  {
+                    alert: 'PostgreSQLPersistentVolumeFillingUp',
+                    annotations: {
+                      description: 'Based on recent sampling, the PersistentVolume claimed by {{\n              $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace\n              }} is expected to fill up within four days. Currently {{ $value |\n              humanizePercentage }} is available.',
+                      runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup',
+                      summary: 'PersistentVolume is filling up.',
+                    },
+                    expr: '(\n            kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}\n              /\n            kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}\n          ) < 0.15\n          and\n          kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0\n          and\n          predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0\n          unless on(namespace, persistentvolumeclaim)\n          kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1\n          unless on(namespace, persistentvolumeclaim)\n          kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1',
+                    'for': '1h',
+                    labels: {
+                      severity: 'warning',
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        },
+      },
+    },
+  },
+  patches: [
+    comp.FromCompositeFieldPathWithTransformSuffix('metadata.labels[crossplane.io/composite]', 'metadata.name', 'prometheusrule'),
+    comp.FromCompositeFieldPathWithTransformPrefix('metadata.labels[crossplane.io/composite]', 'spec.forProvider.manifest.metadata.namespace', 'vshn-postgresql'),
+  ],
+};
+
 local composition(restore=false) =
 
   local metadata = if restore then common.VshnMetaVshn('PostgreSQLRestore', 'standalone', 'false') else common.VshnMetaVshn('PostgreSQL', 'standalone');
@@ -875,6 +928,7 @@ local composition(restore=false) =
                    maintenanceRole,
                    maintenanceRoleBinding,
                    maintenanceJob,
+                   prometheusRule,
                  ] + if pgParams.enableNetworkPolicy == true then [
         networkPolicy,
       ] else [],

diff --git a/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml
@@ -183,6 +183,18 @@ rules:
       - vshnpostgresqls
     verbs:
       - get
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - prometheusrules
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml
@@ -183,6 +183,18 @@ rules:
       - vshnpostgresqls
     verbs:
       - get
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - prometheusrules
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/tests/golden/openshift/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/openshift/appcat/appcat/10_provider_kubernetes.yaml
@@ -185,6 +185,18 @@ rules:
       - vshnpostgresqls
     verbs:
       - get
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - prometheusrules
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/tests/golden/vshn/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/vshn/appcat/appcat/10_provider_kubernetes.yaml
@@ -183,6 +183,18 @@ rules:
       - vshnpostgresqls
     verbs:
       - get
+  - apiGroups:
+      - monitoring.coreos.com
+    resources:
+      - prometheusrules
+    verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+      - create
+      - delete
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml
@@ -986,6 +986,89 @@ spec:
               - fromFieldPath: maintenance.dayOfWeek
           toFieldPath: spec.forProvider.manifest.spec.schedule
           type: CombineFromEnvironment
+    - base:
+        apiVersion: kubernetes.crossplane.io/v1alpha1
+        kind: Object
+        metadata: {}
+        spec:
+          forProvider:
+            manifest:
+              apiVersion: monitoring.coreos.com/v1
+              kind: PrometheusRule
+              metadata:
+                name: postgresql-storage-rules
+              spec:
+                groups:
+                  - name: postgresql-storage
+                    rules:
+                      - alert: PostgreSQLPersistentVolumeFillingUp
+                        annotations:
+                          description: |-
+                            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
+                                          }} in Namespace {{ $labels.namespace }} is only {{ $value |
+                                          humanizePercentage }} free.
+                          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
+                          summary: PersistentVolume is filling up.
+                        expr: |-
+                          (
+                                      kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
+                                        /
+                                      kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
+                                    ) < 0.03
+                                    and
+                                    kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+                        for: 1m
+                        labels:
+                          severity: critical
+                      - alert: PostgreSQLPersistentVolumeFillingUp
+                        annotations:
+                          description: |-
+                            Based on recent sampling, the PersistentVolume claimed by {{
+                                          $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace
+                                          }} is expected to fill up within four days. Currently {{ $value |
+                                          humanizePercentage }} is available.
+                          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
+                          summary: PersistentVolume is filling up.
+                        expr: |-
+                          (
+                                      kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
+                                        /
+                                      kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
+                                    ) < 0.15
+                                    and
+                                    kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
+                                    and
+                                    predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+                        for: 1h
+                        labels:
+                          severity: warning
+          providerConfigRef:
+            name: kubernetes
+      patches:
+        - fromFieldPath: metadata.labels[crossplane.io/composite]
+          toFieldPath: metadata.name
+          transforms:
+            - string:
+                fmt: '%s-prometheusrule'
+                type: Format
+              type: string
+          type: FromCompositeFieldPath
+        - fromFieldPath: metadata.labels[crossplane.io/composite]
+          toFieldPath: spec.forProvider.manifest.metadata.namespace
+          transforms:
+            - string:
+                fmt: vshn-postgresql-%s
+                type: Format
+              type: string
+          type: FromCompositeFieldPath
     - base:
         apiVersion: kubernetes.crossplane.io/v1alpha1
         kind: Object

diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml
@@ -1087,6 +1087,89 @@ spec:
               - fromFieldPath: maintenance.dayOfWeek
           toFieldPath: spec.forProvider.manifest.spec.schedule
           type: CombineFromEnvironment
+    - base:
+        apiVersion: kubernetes.crossplane.io/v1alpha1
+        kind: Object
+        metadata: {}
+        spec:
+          forProvider:
+            manifest:
+              apiVersion: monitoring.coreos.com/v1
+              kind: PrometheusRule
+              metadata:
+                name: postgresql-storage-rules
+              spec:
+                groups:
+                  - name: postgresql-storage
+                    rules:
+                      - alert: PostgreSQLPersistentVolumeFillingUp
+                        annotations:
+                          description: |-
+                            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
+                                          }} in Namespace {{ $labels.namespace }} is only {{ $value |
+                                          humanizePercentage }} free.
+                          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
+                          summary: PersistentVolume is filling up.
+                        expr: |-
+                          (
+                                      kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
+                                        /
+                                      kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
+                                    ) < 0.03
+                                    and
+                                    kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+                        for: 1m
+                        labels:
+                          severity: critical
+                      - alert: PostgreSQLPersistentVolumeFillingUp
+                        annotations:
+                          description: |-
+                            Based on recent sampling, the PersistentVolume claimed by {{
+                                          $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace
+                                          }} is expected to fill up within four days. Currently {{ $value |
+                                          humanizePercentage }} is available.
+                          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup
+                          summary: PersistentVolume is filling up.
+                        expr: |-
+                          (
+                                      kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
+                                        /
+                                      kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}
+                                    ) < 0.15
+                                    and
+                                    kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0
+                                    and
+                                    predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1
+                                    unless on(namespace, persistentvolumeclaim)
+                                    kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1
+                        for: 1h
+                        labels:
+                          severity: warning
+          providerConfigRef:
+            name: kubernetes
+      patches:
+        - fromFieldPath: metadata.labels[crossplane.io/composite]
+          toFieldPath: metadata.name
+          transforms:
+            - string:
+                fmt: '%s-prometheusrule'
+                type: Format
+              type: string
+          type: FromCompositeFieldPath
+        - fromFieldPath: metadata.labels[crossplane.io/composite]
+          toFieldPath: spec.forProvider.manifest.metadata.namespace
+          transforms:
+            - string:
+                fmt: vshn-postgresql-%s
+                type: Format
+              type: string
+          type: FromCompositeFieldPath
     - base:
         apiVersion: kubernetes.crossplane.io/v1alpha1
         kind: Object