From f51fb409dde6f5dfed65d5eca4c51c85c1b6ca66 Mon Sep 17 00:00:00 2001 From: Smaine Kahlouch Date: Sun, 24 Nov 2024 08:29:52 +0100 Subject: [PATCH] chore(oncall): use CNPG instead of RDS --- dagger/go.mod | 6 ++-- dagger/go.sum | 16 +++++------ ...d.yaml => externalsecret-cnpg-oncall.yaml} | 6 ++-- .../grafana-oncall/helmrelease-oncall.yaml | 12 ++++---- .../grafana-oncall/helmrelease-rabbitmq.yaml | 2 +- .../base/grafana-oncall/kustomization.yaml | 2 +- .../base/grafana-oncall/sqlinstance.yaml | 28 ++++++++----------- .../grafana-victoriametrics.yaml | 8 +++--- ...ml => externalsecret-grafana-envvars.yaml} | 6 ++-- .../kustomization.yaml | 4 ++- .../vm-common-helm-values-configmap.yaml | 21 ++++++++++++-- .../cert-manager/openbao-clusterissuer.yaml | 2 +- security/base/zitadel/sqlinstance.yaml | 2 +- tooling/mycluster-0/kustomization.yaml | 1 - 14 files changed, 63 insertions(+), 53 deletions(-) rename observability/base/grafana-oncall/{externalsecret-sqlinstance-password.yaml => externalsecret-cnpg-oncall.yaml} (70%) rename observability/base/victoria-metrics-k8s-stack/{externalsecret-grafana-admin.yaml => externalsecret-grafana-envvars.yaml} (65%) diff --git a/dagger/go.mod b/dagger/go.mod index 1fd26c59..5d37977b 100644 --- a/dagger/go.mod +++ b/dagger/go.mod @@ -7,17 +7,17 @@ toolchain go1.23.2 require github.com/aws/aws-sdk-go v1.55.5 require ( - github.com/99designs/gqlgen v0.17.56 + github.com/99designs/gqlgen v0.17.57 github.com/Khan/genqlient v0.7.0 github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sosodev/duration v1.3.1 // indirect - github.com/vektah/gqlparser/v2 v2.5.19 + github.com/vektah/gqlparser/v2 v2.5.20 go.opentelemetry.io/otel v1.32.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.8.0 go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.8.0 diff --git a/dagger/go.sum b/dagger/go.sum index 81f59bc9..1f931274 100644 --- a/dagger/go.sum +++ b/dagger/go.sum @@ -1,5 +1,5 @@ -github.com/99designs/gqlgen v0.17.56 h1:+J42ARAHvnysH6klO9Wq+tCsGF32cpAgU3SyF0VRJtI= -github.com/99designs/gqlgen v0.17.56/go.mod h1:rmB6vLvtL8uf9F9w0/irJ5alBkD8DJvj35ET31BKbtY= +github.com/99designs/gqlgen v0.17.57 h1:Ak4p60BRq6QibxY0lEc0JnQhDurfhxA67sp02lMjmPc= +github.com/99designs/gqlgen v0.17.57/go.mod h1:Jx61hzOSTcR4VJy/HFIgXiQ5rJ0Ypw8DxWLjbYDAUw0= github.com/Khan/genqlient v0.7.0 h1:GZ1meyRnzcDTK48EjqB8t3bcfYvHArCUUvgOwpz1D4w= github.com/Khan/genqlient v0.7.0/go.mod h1:HNyy3wZvuYwmW3Y7mkoQLZsa/R5n5yIRajS1kPBvSFM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= @@ -22,8 +22,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 h1:ad0vkEBuk23VJzZR9nkLVG0YAoN9coASF1GusYX6AlU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0/go.mod h1:igFoXX2ELCW06bol23DWPB5BEWfZISOzSP5K2sbLea0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -39,10 +39,10 @@ github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vektah/gqlparser/v2 v2.5.19 h1:bhCPCX1D4WWzCDvkPl4+TP1N8/kLrWnp43egplt7iSg= -github.com/vektah/gqlparser/v2 v2.5.19/go.mod h1:y7kvl5bBlDeuWIvLtA9849ncyvx6/lj06RsMrEjVy3U= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vektah/gqlparser/v2 v2.5.20 h1:kPaWbhBntxoZPaNdBaIPT1Kh0i1b/onb5kXgEdP5JCo= +github.com/vektah/gqlparser/v2 v2.5.20/go.mod h1:xMl+ta8a5M1Yo1A1Iwt/k7gSpscwSnHZdw7tfhEGfTM= go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.0.0-20240518090000-14441aefdf88 h1:oM0GTNKGlc5qHctWeIGTVyda4iFFalOzMZ3Ehj5rwB4= diff --git a/observability/base/grafana-oncall/externalsecret-sqlinstance-password.yaml b/observability/base/grafana-oncall/externalsecret-cnpg-oncall.yaml similarity index 70% rename from observability/base/grafana-oncall/externalsecret-sqlinstance-password.yaml rename to observability/base/grafana-oncall/externalsecret-cnpg-oncall.yaml index 0117d662..a220e331 100644 --- a/observability/base/grafana-oncall/externalsecret-sqlinstance-password.yaml +++ b/observability/base/grafana-oncall/externalsecret-cnpg-oncall.yaml @@ -1,12 +1,12 @@ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: oncall-sqlinstance-password + name: xplane-oncall-cnpg-oncall spec: dataFrom: - extract: conversionStrategy: Default - key: observability/grafana/oncall-sqlinstance + key: cnpg/xplane-oncall/oncall refreshInterval: 20m secretStoreRef: kind: ClusterSecretStore @@ -14,4 +14,4 @@ spec: target: creationPolicy: Owner deletionPolicy: Retain - name: oncall-pg-masterpassword + name: xplane-oncall-cnpg-oncall diff --git a/observability/base/grafana-oncall/helmrelease-oncall.yaml b/observability/base/grafana-oncall/helmrelease-oncall.yaml index c05381c4..296422d4 100644 --- a/observability/base/grafana-oncall/helmrelease-oncall.yaml +++ b/observability/base/grafana-oncall/helmrelease-oncall.yaml @@ -14,7 +14,7 @@ spec: kind: HelmRepository name: grafana namespace: observability - version: "1.11.5" + version: "1.13.5" interval: 5m0s timeout: 15m install: @@ -28,8 +28,8 @@ spec: replicaCount: 1 resources: limits: - cpu: 200m - memory: 256Mi + cpu: 500m + memory: 768Mi celery: replicaCount: 1 @@ -66,14 +66,12 @@ spec: enabled: false externalPostgresql: - host: xplane-oncall-rds-service + host: xplane-oncall-cnpg-cluster-rw port: 5432 db_name: "oncall" user: "oncall" - existingSecret: "xplane-oncall-owner-oncall" + existingSecret: "xplane-oncall-cnpg-oncall" passwordKey: "password" - options: >- - sslmode=require rabbitmq: enabled: false diff --git a/observability/base/grafana-oncall/helmrelease-rabbitmq.yaml b/observability/base/grafana-oncall/helmrelease-rabbitmq.yaml index 7b77d8d2..26b5f069 100644 --- a/observability/base/grafana-oncall/helmrelease-rabbitmq.yaml +++ b/observability/base/grafana-oncall/helmrelease-rabbitmq.yaml @@ -14,7 +14,7 @@ spec: kind: HelmRepository name: bitnami namespace: flux-system - version: "15.0.3" + version: "15.0.6" interval: 5m0s timeout: 15m install: diff --git a/observability/base/grafana-oncall/kustomization.yaml b/observability/base/grafana-oncall/kustomization.yaml index c637c729..3c87f6d4 100644 --- a/observability/base/grafana-oncall/kustomization.yaml +++ b/observability/base/grafana-oncall/kustomization.yaml @@ -4,9 +4,9 @@ namespace: observability resources: - externalsecret-admin.yaml + - externalsecret-cnpg-oncall.yaml - externalsecret-rabbitmq.yaml - externalsecret-slackapp.yaml - - externalsecret-sqlinstance-password.yaml - externalsecret-valkey.yaml - helmrelease-oncall.yaml - helmrelease-rabbitmq.yaml diff --git a/observability/base/grafana-oncall/sqlinstance.yaml b/observability/base/grafana-oncall/sqlinstance.yaml index f88716e6..10f94113 100644 --- a/observability/base/grafana-oncall/sqlinstance.yaml +++ b/observability/base/grafana-oncall/sqlinstance.yaml @@ -2,21 +2,15 @@ apiVersion: cloud.ogenki.io/v1alpha1 kind: SQLInstance metadata: name: xplane-oncall - namespace: tooling + namespace: "observability" spec: - parameters: - engine: postgres - engineVersion: "16" - size: small - storageGB: 20 - databases: - - owner: oncall - name: oncall - passwordSecretRef: - namespace: tooling - name: oncall-pg-masterpassword - key: password - compositionRef: - name: xsqlinstances.cloud.ogenki.io - writeConnectionSecretToRef: - name: xplane-oncall-rds + size: "small" + storageGB: 20 + databases: + - owner: "oncall" + name: "oncall" + cnpg: + instances: 1 + backup: + schedule: "0 0 * * *" + bucketName: "eu-west-3-ogenki-cnpg-backups" diff --git a/observability/base/grafana-operator/grafana-victoriametrics.yaml b/observability/base/grafana-operator/grafana-victoriametrics.yaml index d38f9cad..651fc58c 100644 --- a/observability/base/grafana-operator/grafana-victoriametrics.yaml +++ b/observability/base/grafana-operator/grafana-victoriametrics.yaml @@ -8,8 +8,8 @@ spec: external: url: http://victoria-metrics-k8s-stack-grafana adminPassword: - name: victoria-metrics-k8s-stack-grafana-admin - key: admin-password + name: victoria-metrics-k8s-stack-grafana-envvars + key: GF_SECURITY_ADMIN_PASSWORD adminUser: - name: victoria-metrics-k8s-stack-grafana-admin - key: admin-user + name: victoria-metrics-k8s-stack-grafana-envvars + key: GF_SECURITY_ADMIN_USER diff --git a/observability/base/victoria-metrics-k8s-stack/externalsecret-grafana-admin.yaml b/observability/base/victoria-metrics-k8s-stack/externalsecret-grafana-envvars.yaml similarity index 65% rename from observability/base/victoria-metrics-k8s-stack/externalsecret-grafana-admin.yaml rename to observability/base/victoria-metrics-k8s-stack/externalsecret-grafana-envvars.yaml index d179b67a..d50bc27e 100644 --- a/observability/base/victoria-metrics-k8s-stack/externalsecret-grafana-admin.yaml +++ b/observability/base/victoria-metrics-k8s-stack/externalsecret-grafana-envvars.yaml @@ -1,13 +1,13 @@ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret metadata: - name: victoria-metrics-k8s-stack-grafana-admin + name: victoria-metrics-k8s-stack-grafana-envvars namespace: observability spec: dataFrom: - extract: conversionStrategy: Default - key: observability/victoria-metrics-k8s-stack/grafana-admin + key: observability/victoria-metrics-k8s-stack/grafana-envvars refreshInterval: 1h secretStoreRef: kind: ClusterSecretStore @@ -15,4 +15,4 @@ spec: target: creationPolicy: Owner deletionPolicy: Retain - name: victoria-metrics-k8s-stack-grafana-admin + name: victoria-metrics-k8s-stack-grafana-envvars diff --git a/observability/base/victoria-metrics-k8s-stack/kustomization.yaml b/observability/base/victoria-metrics-k8s-stack/kustomization.yaml index 8f7ba8aa..fdcb83a2 100644 --- a/observability/base/victoria-metrics-k8s-stack/kustomization.yaml +++ b/observability/base/victoria-metrics-k8s-stack/kustomization.yaml @@ -3,7 +3,7 @@ kind: Kustomization resources: - externalsecret-alertmanager-slack-app.yaml - - externalsecret-grafana-admin.yaml + - externalsecret-grafana-envvars.yaml # HttpRoutes - httproute-grafana.yaml @@ -15,7 +15,9 @@ resources: - vmservicecrapes - vmrules + # Common Helm values for both single and cluster - vm-common-helm-values-configmap.yaml + # Choose between single or cluster helm release # VM Single diff --git a/observability/base/victoria-metrics-k8s-stack/vm-common-helm-values-configmap.yaml b/observability/base/victoria-metrics-k8s-stack/vm-common-helm-values-configmap.yaml index b23b1819..72cba277 100644 --- a/observability/base/victoria-metrics-k8s-stack/vm-common-helm-values-configmap.yaml +++ b/observability/base/victoria-metrics-k8s-stack/vm-common-helm-values-configmap.yaml @@ -79,8 +79,6 @@ data: url: "{{ .CommonAnnotations.link_url }}" grafana: - admin: - existingSecret: "victoria-metrics-k8s-stack-grafana-admin" sidecar: datasources: enabled: true @@ -93,6 +91,25 @@ data: apiVersion: 1 providers: [] dashboards: {} + plugins: + - "grafana-oncall-app" + "grafana.ini": + feature_toggles: + enable: externalServiceAccounts + accessControlOnCall: 'false' + server: + root_url: "https://grafana.priv.${domain_name}" + domain: "grafana.priv.${domain_name}" + auth.generic_oauth: + enabled: true + name: "Zitadel" + allow_sign_up: true + scopes: "openid profile email" + auth_url: "https://auth.${domain_name}/oauth/v2/authorize" + token_url: "https://auth.${domain_name}/oauth/v2/token" + api_url: "https://auth.${domain_name}/oidc/v1/userinfo" + # role_attribute_path: "contains(groups[*], 'admin-group') && 'Admin' || 'Viewer'" + envFromSecret: "victoria-metrics-k8s-stack-grafana-envvars" extraInitContainers: - name: "load-vm-ds-plugin" image: "curlimages/curl:8.11.0" diff --git a/security/base/cert-manager/openbao-clusterissuer.yaml b/security/base/cert-manager/openbao-clusterissuer.yaml index 379e9dce..164f78f4 100644 --- a/security/base/cert-manager/openbao-clusterissuer.yaml +++ b/security/base/cert-manager/openbao-clusterissuer.yaml @@ -11,7 +11,7 @@ spec: auth: appRole: path: approle - roleId: f52c783d-0259-86a4-c80d-2380a9cc443f # !! This value changes each time I recreate the whole platform + roleId: e33fb753-cb9a-c9e7-9796-779602e70542 # !! This value changes each time I recreate the whole platform secretRef: name: cert-manager-openbao-approle key: secret_id diff --git a/security/base/zitadel/sqlinstance.yaml b/security/base/zitadel/sqlinstance.yaml index 5f12f003..6acb2ba5 100644 --- a/security/base/zitadel/sqlinstance.yaml +++ b/security/base/zitadel/sqlinstance.yaml @@ -10,7 +10,7 @@ spec: createSuperuser: true objectStoreRecovery: bucketName: "eu-west-3-ogenki-cnpg-backups" - path: "zitadel-20241116" + path: "zitadel-20241201" backup: schedule: "0 0 * * *" bucketName: "eu-west-3-ogenki-cnpg-backups" diff --git a/tooling/mycluster-0/kustomization.yaml b/tooling/mycluster-0/kustomization.yaml index 14f0a3a7..507bfb18 100644 --- a/tooling/mycluster-0/kustomization.yaml +++ b/tooling/mycluster-0/kustomization.yaml @@ -4,6 +4,5 @@ kind: Kustomization resources: - ../base/harbor - ../base/headlamp - # Uncomment the following resources to include them in the kustomization # - ../base/dagger-engine # - ../base/gha-runners