diff --git a/component/provider.jsonnet b/component/provider.jsonnet index b3f7fc729..befd03817 100644 --- a/component/provider.jsonnet +++ b/component/provider.jsonnet @@ -168,6 +168,11 @@ local controllerConfigRef(config) = resources: [ 'vshnpostgresqls' ], verbs: [ 'get' ], }, + { + apiGroups: [ 'monitoring.coreos.com' ], + resources: [ 'prometheusrules' ], + verbs: [ 'get', 'list', 'watch', 'update', 'patch', 'create', 'delete' ], + }, ], }; local rolebinding = kube.ClusterRoleBinding('crossplane:provider:provider-kubernetes:system:custom') { diff --git a/component/vshn_postgres.jsonnet b/component/vshn_postgres.jsonnet index 00db6dc0f..a78b9ff7a 100644 --- a/component/vshn_postgres.jsonnet +++ b/component/vshn_postgres.jsonnet @@ -842,6 +842,59 @@ local maintenanceJob = { ] + convertToCron(), }; +local prometheusRule = { + base: comp.KubeObject('monitoring.coreos.com/v1', 'PrometheusRule') + { + spec+: { + forProvider+: { + manifest+: { + metadata: { + name: 'postgresql-storage-rules', + }, + spec: { + groups: [ + { + name: 'postgresql-storage', + rules: [ + { + alert: 'PostgreSQLPersistentVolumeFillingUp', + annotations: { + description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim\n }} in Namespace {{ $labels.namespace }} is only {{ $value |\n humanizePercentage }} free.', + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup', + summary: 'PersistentVolume is filling up.', + }, + expr: '(\n kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}\n /\n kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}\n ) < 0.03\n and\n kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0\n unless on(namespace, persistentvolumeclaim)\n kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1\n unless on(namespace, persistentvolumeclaim)\n kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1', + 'for': '1m', + labels: { + severity: 'critical', + }, + }, + { + alert: 'PostgreSQLPersistentVolumeFillingUp', + annotations: { + description: 'Based on recent sampling, the PersistentVolume claimed by {{\n $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace\n }} is expected to fill up within four days. Currently {{ $value |\n humanizePercentage }} is available.', + runbook_url: 'https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup', + summary: 'PersistentVolume is filling up.', + }, + expr: '(\n kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}\n /\n kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"}\n ) < 0.15\n and\n kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0\n and\n predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0\n unless on(namespace, persistentvolumeclaim)\n kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1\n unless on(namespace, persistentvolumeclaim)\n kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1', + 'for': '1h', + labels: { + severity: 'warning', + }, + }, + ], + }, + ], + }, + }, + }, + }, + }, + patches: [ + comp.FromCompositeFieldPathWithTransformSuffix('metadata.labels[crossplane.io/composite]', 'metadata.name', 'prometheusrule'), + comp.FromCompositeFieldPathWithTransformPrefix('metadata.labels[crossplane.io/composite]', 'spec.forProvider.manifest.metadata.namespace', 'vshn-postgresql'), + ], +}; + local composition(restore=false) = local metadata = if restore then common.VshnMetaVshn('PostgreSQLRestore', 'standalone', 'false') else common.VshnMetaVshn('PostgreSQL', 'standalone'); @@ -875,6 +928,7 @@ local composition(restore=false) = maintenanceRole, maintenanceRoleBinding, maintenanceJob, + prometheusRule, ] + if pgParams.enableNetworkPolicy == true then [ networkPolicy, ] else [], diff --git a/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml index a5ec2651f..0391e69db 100644 --- a/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml +++ b/tests/golden/cloudscale/appcat/appcat/10_provider_kubernetes.yaml @@ -183,6 +183,18 @@ rules: - vshnpostgresqls verbs: - get + - apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - get + - list + - watch + - update + - patch + - create + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml index a5ec2651f..0391e69db 100644 --- a/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml +++ b/tests/golden/exoscale/appcat/appcat/10_provider_kubernetes.yaml @@ -183,6 +183,18 @@ rules: - vshnpostgresqls verbs: - get + - apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - get + - list + - watch + - update + - patch + - create + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/tests/golden/openshift/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/openshift/appcat/appcat/10_provider_kubernetes.yaml index 86f83d069..845aa640c 100644 --- a/tests/golden/openshift/appcat/appcat/10_provider_kubernetes.yaml +++ b/tests/golden/openshift/appcat/appcat/10_provider_kubernetes.yaml @@ -185,6 +185,18 @@ rules: - vshnpostgresqls verbs: - get + - apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - get + - list + - watch + - update + - patch + - create + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/tests/golden/vshn/appcat/appcat/10_provider_kubernetes.yaml b/tests/golden/vshn/appcat/appcat/10_provider_kubernetes.yaml index a5ec2651f..0391e69db 100644 --- a/tests/golden/vshn/appcat/appcat/10_provider_kubernetes.yaml +++ b/tests/golden/vshn/appcat/appcat/10_provider_kubernetes.yaml @@ -183,6 +183,18 @@ rules: - vshnpostgresqls verbs: - get + - apiGroups: + - monitoring.coreos.com + resources: + - prometheusrules + verbs: + - get + - list + - watch + - update + - patch + - create + - delete --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml index 1b88989a4..75c2000b3 100644 --- a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml +++ b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgres.yaml @@ -986,6 +986,89 @@ spec: - fromFieldPath: maintenance.dayOfWeek toFieldPath: spec.forProvider.manifest.spec.schedule type: CombineFromEnvironment + - base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + metadata: {} + spec: + forProvider: + manifest: + apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: postgresql-storage-rules + spec: + groups: + - name: postgresql-storage + rules: + - alert: PostgreSQLPersistentVolumeFillingUp + annotations: + description: |- + The PersistentVolume claimed by {{ $labels.persistentvolumeclaim + }} in Namespace {{ $labels.namespace }} is only {{ $value | + humanizePercentage }} free. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical + - alert: PostgreSQLPersistentVolumeFillingUp + annotations: + description: |- + Based on recent sampling, the PersistentVolume claimed by {{ + $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace + }} is expected to fill up within four days. Currently {{ $value | + humanizePercentage }} is available. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning + providerConfigRef: + name: kubernetes + patches: + - fromFieldPath: metadata.labels[crossplane.io/composite] + toFieldPath: metadata.name + transforms: + - string: + fmt: '%s-prometheusrule' + type: Format + type: string + type: FromCompositeFieldPath + - fromFieldPath: metadata.labels[crossplane.io/composite] + toFieldPath: spec.forProvider.manifest.metadata.namespace + transforms: + - string: + fmt: vshn-postgresql-%s + type: Format + type: string + type: FromCompositeFieldPath - base: apiVersion: kubernetes.crossplane.io/v1alpha1 kind: Object diff --git a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml index 39703fc98..4f06f0acd 100644 --- a/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml +++ b/tests/golden/vshn/appcat/appcat/21_composition_vshn_postgresrestore.yaml @@ -1087,6 +1087,89 @@ spec: - fromFieldPath: maintenance.dayOfWeek toFieldPath: spec.forProvider.manifest.spec.schedule type: CombineFromEnvironment + - base: + apiVersion: kubernetes.crossplane.io/v1alpha1 + kind: Object + metadata: {} + spec: + forProvider: + manifest: + apiVersion: monitoring.coreos.com/v1 + kind: PrometheusRule + metadata: + name: postgresql-storage-rules + spec: + groups: + - name: postgresql-storage + rules: + - alert: PostgreSQLPersistentVolumeFillingUp + annotations: + description: |- + The PersistentVolume claimed by {{ $labels.persistentvolumeclaim + }} in Namespace {{ $labels.namespace }} is only {{ $value | + humanizePercentage }} free. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} + ) < 0.03 + and + kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1m + labels: + severity: critical + - alert: PostgreSQLPersistentVolumeFillingUp + annotations: + description: |- + Based on recent sampling, the PersistentVolume claimed by {{ + $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace + }} is expected to fill up within four days. Currently {{ $value | + humanizePercentage }} is available. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/kubernetes/kubepersistentvolumefillingup + summary: PersistentVolume is filling up. + expr: |- + ( + kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} + / + kubelet_volume_stats_capacity_bytes{job="kubelet", metrics_path="/metrics"} + ) < 0.15 + and + kubelet_volume_stats_used_bytes{job="kubelet", metrics_path="/metrics"} > 0 + and + predict_linear(kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}[6h], 4 * 24 * 3600) < 0 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_access_mode{ access_mode="ReadOnlyMany"} == 1 + unless on(namespace, persistentvolumeclaim) + kube_persistentvolumeclaim_labels{label_excluded_from_alerts="true"} == 1 + for: 1h + labels: + severity: warning + providerConfigRef: + name: kubernetes + patches: + - fromFieldPath: metadata.labels[crossplane.io/composite] + toFieldPath: metadata.name + transforms: + - string: + fmt: '%s-prometheusrule' + type: Format + type: string + type: FromCompositeFieldPath + - fromFieldPath: metadata.labels[crossplane.io/composite] + toFieldPath: spec.forProvider.manifest.metadata.namespace + transforms: + - string: + fmt: vshn-postgresql-%s + type: Format + type: string + type: FromCompositeFieldPath - base: apiVersion: kubernetes.crossplane.io/v1alpha1 kind: Object