From 4462ed70b89b6ca5e87a435e78623394c82d0ac8 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Wed, 20 Sep 2023 21:42:22 +0530 Subject: [PATCH 1/5] adding dsnexec as a sidecar --- cmd/manager/main.go | 46 +++++++-- config/dsnexec/dsnexecsidecar.json | 35 +++++++ helm/db-controller/templates/certs.yaml | 2 +- helm/db-controller/templates/deployment.yaml | 10 +- helm/db-controller/templates/webhook.yaml | 28 +++++- helm/db-controller/values.yaml | 6 ++ webhook/webhook-dsnexec.go | 99 ++++++++++++++++++++ 7 files changed, 212 insertions(+), 14 deletions(-) create mode 100644 config/dsnexec/dsnexecsidecar.json create mode 100644 webhook/webhook-dsnexec.go diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 849d3622..08c84533 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -82,10 +82,12 @@ func main() { var metricsPort int var enableLeaderElection bool var configFile string - var sidecarConfigPath string + var dBProxySidecarConfigPath string + var dsnExecSidecarConfigPath string var probeAddr string var probePort int var enableDBProxyWebhook bool + var enableDsnExecWebhook bool var dbIdentifierPrefix string var class string var metricsDepYamlPath string @@ -101,13 +103,18 @@ func main() { "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&configFile, "config-file", "/etc/config/config.yaml", "Database connection string to with root credentials.") - flag.StringVar(&sidecarConfigPath, "sidecar-config-path", "/etc/config/sidecar.yaml", "Mutating webhook sidecar configuration.") + flag.StringVar(&dBProxySidecarConfigPath, "db-proxy-sidecar-config-path", "/etc/config/sidecar.yaml", "Mutating webhook sidecar configuration.") + flag.StringVar(&dsnExecSidecarConfigPath, "dsnexec-sidecar-config-path", "/etc/config/sidecar.yaml", "Mutating webhook sidecar configuration.") flag.StringVar(&metricsDepYamlPath, "metrics-dep-yaml", "/config/postgres-exporter/deployment.yaml", "path to the metrics deployment yaml") flag.StringVar(&metricsConfigYamlPath, "metrics-config-yaml", "/config/postgres-exporter/config.yaml", "path to the metrics config yaml") flag.BoolVar(&enableDBProxyWebhook, "enable-db-proxy", false, "Enable DB Proxy webhook. "+ "Enabling this option will cause the db-controller to inject db proxy pod into pods "+ "with the infoblox.com/db-secret-path annotation set.") + flag.BoolVar(&enableDsnExecWebhook, "enable-dsnexec", false, + "Enable Dsnexec webhook. "+ + "Enabling this option will cause the db-controller to inject dsnexec container into pods "+ + "with the infoblox.com/remote-db-dsn-secret and infoblox.com/dsnexec-config-secret annotations set.") opts := zap.Options{ Development: false, TimeEncoder: zapcore.RFC3339NanoTimeEncoder, @@ -166,21 +173,22 @@ func main() { os.Exit(1) } + webHookServer := mgr.GetWebhookServer() + + webHookServer.Port = 7443 + webHookServer.CertDir = "./certs/" + if enableDBProxyWebhook { - webHookServer := mgr.GetWebhookServer() - webHookServer.Port = 7443 - webHookServer.CertDir = "./certs/" + cfg, err := dbwebhook.ParseConfig(dBProxySidecarConfigPath) - cfg, err := dbwebhook.ParseConfig(sidecarConfigPath) if err != nil { setupLog.Error(err, "could not parse db proxy sidecar configuration") os.Exit(1) } + setupLog.Info("Parsed db proxy conig:", "dbproxysidecarconfig", cfg) - setupLog.Info("Parsed db proxy config:", "dbproxysidecarconfig", cfg) - - setupLog.Info("registering with webhook server") + setupLog.Info("registering with webhook server for DbProxy") webHookServer.Register("/mutate", &webhook.Admission{ Handler: &dbwebhook.DBProxyInjector{ Name: "DB Proxy", @@ -190,6 +198,26 @@ func main() { }) } + if enableDsnExecWebhook { + + cfg, err := dbwebhook.ParseConfig(dsnExecSidecarConfigPath) + + if err != nil { + setupLog.Error(err, "could not parse dsnexec sidecar configuration") + os.Exit(1) + } + setupLog.Info("Parsed dsnexec conig:", "dsnexecsidecarconfig", cfg) + + setupLog.Info("registering with webhook server for DsnExec") + webHookServer.Register("/mutate-dsnexec", &webhook.Admission{ + Handler: &dbwebhook.DsnExecInjector{ + Name: "Dsnexec", + Client: mgr.GetClient(), + DsnExecSidecarConfig: cfg, + }, + }) + } + setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") diff --git a/config/dsnexec/dsnexecsidecar.json b/config/dsnexec/dsnexecsidecar.json new file mode 100644 index 00000000..44a43675 --- /dev/null +++ b/config/dsnexec/dsnexecsidecar.json @@ -0,0 +1,35 @@ +{ + "containers": [ + { + "imagePullPolicy": "IfNotPresent", + "args": ["run","-c","/var/run/dsn-exec/config.yaml"], + "name": "dsn-exec", + "volumeMounts": [ + { + "mountPath": "/var/run/db-dsn", + "name": "remote-db-dsn-volume" + }, + { + "mountPath": "/var/run/dsn-exec", + "name": "dsnexec-config-volume" + } + ] + } + ], + "volumes": [ + { + "name": "remote-db-dsn-volume", + "secret": { + "optional": false, + "secretName": "..." + } + }, + { + "name": "dsnexec-config-volume", + "secret": { + "optional": false, + "secretName": "..." + } + } + ] +} \ No newline at end of file diff --git a/helm/db-controller/templates/certs.yaml b/helm/db-controller/templates/certs.yaml index c9c10e38..1c570c15 100644 --- a/helm/db-controller/templates/certs.yaml +++ b/helm/db-controller/templates/certs.yaml @@ -1,4 +1,4 @@ -{{- if .Values.dbproxy.enabled }} +{{- if or ( .Values.dbproxy.enabled ) ( .Values.dsnexec.enabled ) }} apiVersion: cert-manager.io/v1 kind: Issuer metadata: diff --git a/helm/db-controller/templates/deployment.yaml b/helm/db-controller/templates/deployment.yaml index c65e9253..33147531 100644 --- a/helm/db-controller/templates/deployment.yaml +++ b/helm/db-controller/templates/deployment.yaml @@ -54,6 +54,8 @@ spec: fieldPath: metadata.namespace - name: DBPROXY_IMAGE value: "{{ .Values.dbproxy.image.repository }}:{{ .Values.dbproxy.image.tag | default .Chart.AppVersion }}" + - name: DSNEXEC_IMAGE + value: "{{ .Values.dsnexec.image.repository }}:{{ .Values.dsnexec.image.tag | default .Chart.AppVersion }}" args: - --metrics-addr={{ .Values.metrics.address }} - --metrics-port={{ .Values.metrics.port }} @@ -61,8 +63,10 @@ spec: - --health-probe-port={{ .Values.healthProbe.port }} - --enable-leader-election - --enable-db-proxy={{ .Values.dbproxy.enabled }} + - --enable-dsnexec={{ .Values.dsnexec.enabled }} - --config-file=/etc/config/config.yaml - - --sidecar-config-path=config/dbproxy/dbproxysidecar.json + - --db-proxy-sidecar-config-path=config/dbproxy/dbproxysidecar.json + - --dsnexec-sidecar-config-path=config/dsnexec/dsnexecsidecar.json - --db-identifier-prefix={{ tpl .Values.db.identifier.prefix . }} - --class={{ .Values.dbController.class }} {{ if .Values.zapLogger.develMode }} @@ -96,7 +100,7 @@ spec: mountPath: /pg-temp - name: config-volume mountPath: /etc/config - {{- if .Values.dbproxy.enabled }} + {{- if or ( .Values.dbproxy.enabled ) ( .Values.dsnexec.enabled ) }} - name: dbproxycert mountPath: /certs readOnly: true @@ -107,7 +111,7 @@ spec: - name: config-volume configMap: name: {{ include "db-controller.name" . }}-config - {{- if .Values.dbproxy.enabled }} + {{- if or ( .Values.dbproxy.enabled ) ( .Values.dsnexec.enabled ) }} - name: dbproxycert secret: secretName: {{ include "db-controller.fullname" . }} diff --git a/helm/db-controller/templates/webhook.yaml b/helm/db-controller/templates/webhook.yaml index 8e4c8681..d1ed3c11 100644 --- a/helm/db-controller/templates/webhook.yaml +++ b/helm/db-controller/templates/webhook.yaml @@ -1,4 +1,4 @@ -{{- if .Values.dbproxy.enabled }} +{{- if or ( .Values.dbproxy.enabled ) ( .Values.dsnexec.enabled ) }} apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: @@ -6,6 +6,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "db-controller.fullname" . }} webhooks: +{{- if .Values.dbproxy.enabled }} - clientConfig: caBundle: Cg== service: @@ -29,3 +30,28 @@ webhooks: - pods scope: "Namespaced" {{- end }} +{{- if .Values.dsnexec.enabled }} +- clientConfig: + caBundle: Cg== + service: + name: {{ include "db-controller.fullname" . }} + path: /mutate-dsnexec + port: 7443 + namespace: {{ .Release.Namespace }} + sideEffects: None + admissionReviewVersions: ["v1"] + failurePolicy: Ignore + name: dsnexec-injector.infoblox.com + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods + scope: "Namespaced" +{{- end }} +{{- end }} diff --git a/helm/db-controller/values.yaml b/helm/db-controller/values.yaml index 54774ae2..163b4848 100644 --- a/helm/db-controller/values.yaml +++ b/helm/db-controller/values.yaml @@ -106,6 +106,12 @@ dbproxy: # Overrides the image tag whose default is the chart appVersion. tag: "" +dsnexec: + enabled: true + image: + repository: "" + tag: "" + zapLogger: develMode: false level: info diff --git a/webhook/webhook-dsnexec.go b/webhook/webhook-dsnexec.go new file mode 100644 index 00000000..64a7e21c --- /dev/null +++ b/webhook/webhook-dsnexec.go @@ -0,0 +1,99 @@ +package hook + +import ( + "context" + "encoding/json" + "net/http" + "os" + "strconv" + + corev1 "k8s.io/api/core/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +// DsnExecInjector annotates Pods +type DsnExecInjector struct { + Name string + Client client.Client + decoder *admission.Decoder + DsnExecSidecarConfig *Config +} + +var ( + dsnexecLog = ctrl.Log.WithName("dsnexec-controller") + sidecarImageForDsnExec = os.Getenv("DSNEXEC_IMAGE") +) + +func dsnExecSideCardInjectionRequired(pod *corev1.Pod) (bool, string, string) { + remoteDbSecretName, ok := pod.Annotations["infoblox.com/remote-db-dsn-secret"] + dsnExecConfigSecret, ok2 := pod.Annotations["infoblox.com/dsnexec-config-secret"] + + if !ok || !ok2 { + dsnexecLog.Info("either or both remote-db-dsn-secret, dsnexec-config-secret can not be seen in the annotations.", pod.Name) + return false, "", "" + } + + alreadyInjected, err := strconv.ParseBool(pod.Annotations["infoblox.com/dsnexec-injected"]) + + if err == nil && alreadyInjected { + dsnexecLog.Info("DsnExec sidecar already injected: ", pod.Name, remoteDbSecretName, dsnExecConfigSecret) + return false, remoteDbSecretName, dsnExecConfigSecret + } + + dsnexecLog.Info("DsnExec sidecar Injection required: ", pod.Name, remoteDbSecretName, dsnExecConfigSecret) + + return true, remoteDbSecretName, dsnExecConfigSecret +} + +// DsnExecInjector adds an annotation to every incoming pods. +func (dbpi *DsnExecInjector) Handle(ctx context.Context, req admission.Request) admission.Response { + pod := &corev1.Pod{} + + err := dbpi.decoder.Decode(req, pod) + if err != nil { + dsnexecLog.Info("Sdecar-Injector: cannot decode") + return admission.Errored(http.StatusBadRequest, err) + } + + if pod.Annotations == nil { + pod.Annotations = map[string]string{} + } + + shoudInjectDsnExec, remoteDbSecretName, dsnExecConfigSecret := dsnExecSideCardInjectionRequired(pod) + + if shoudInjectDsnExec { + dsnexecLog.Info("Injecting sidecar...") + + dbpi.DsnExecSidecarConfig.Containers[0].Image = sidecarImageForDsnExec + dbpi.DsnExecSidecarConfig.Volumes[0].Secret.SecretName = remoteDbSecretName + dbpi.DsnExecSidecarConfig.Volumes[1].Secret.SecretName = dsnExecConfigSecret + + pod.Spec.Volumes = append(pod.Spec.Volumes, dbpi.DsnExecSidecarConfig.Volumes...) + pod.Spec.Containers = append(pod.Spec.Containers, dbpi.DsnExecSidecarConfig.Containers...) + + pod.Annotations["infoblox.com/dsnexec-injected"] = "true" + + dsnexecLog.Info("sidecar ontainer for ", dbpi.Name, " injected.", pod.Name, pod.APIVersion) + + } else { + dsnexecLog.Info("dsnexec sidecar not needed.", pod.Name, pod.APIVersion) + } + + marshaledPod, err := json.Marshal(pod) + + if err != nil { + dsnexecLog.Info("dsnexec sidecar injection: cannot marshal") + return admission.Errored(http.StatusInternalServerError, err) + } + + return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) +} + +// DsnExecInjector implements admission.DecoderInjector. +// InjectDecoder injects the decoder. +func (dbpi *DsnExecInjector) InjectDecoder(d *admission.Decoder) error { + dbpi.decoder = d + return nil +} From 897ee891367ce5235af3b6afcec9c63446d30767 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Thu, 21 Sep 2023 00:35:21 +0530 Subject: [PATCH 2/5] changes in makefile to add dsnexec docker commands --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index 41c015ca..23f4c65f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ REGISTRY ?= ghcr.io/infobloxopen # image name IMAGE_NAME ?= db-controller DBPROXY_IMAGE_NAME ?= dbproxy +DSNEXEC_IMAGE_NAME ?= dsnexec # commit tag info from git repo GIT_COMMIT := $(shell git describe --always || echo pre-commit) # image tag @@ -10,6 +11,7 @@ TAG ?= ${GIT_COMMIT} # Image Path to use all building/pushing image targets IMG_PATH ?= ${REGISTRY}/${IMAGE_NAME} DBPROXY_IMG_PATH ?= ${REGISTRY}/${DBPROXY_IMAGE_NAME} +DSNEXEC_IMG_PATH ?= ${REGISTRY}/${DSNEXEC_IMAGE_NAME} GOBIN := ~/go/bin K8S_VERSION := 1.24 # ACK_GINKGO_DEPRECATIONS := 1.16.5 @@ -143,6 +145,8 @@ test: manifests generate fmt vet envtest ## Run tests. build: generate fmt vet ## Build manager binary. cd cmd/manager && go build -o ../../bin/manager main.go cd dbproxy && go build -o ../bin/dbproxy + cd dsnexec && go build -o ../bin/dsnexec + .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. @@ -160,6 +164,10 @@ docker-buildx: generate fmt vet manifests ## Build and optionally push a multi-a docker-build-dbproxy: cd dbproxy && docker build -t ${DBPROXY_IMG_PATH}:${TAG} . +docker-build-dsnexec: + cd dsnexec && docker build -t ${DSNEXEC_IMG_PATH}:${TAG} . + + .PHONY: docker-build docker-build: .image-${TAG} #test ## Build docker image with the manager. @@ -171,6 +179,9 @@ docker-build: .image-${TAG} #test ## Build docker image with the manager. docker-push-dbproxy: docker-build-dbproxy docker push ${DBPROXY_IMG_PATH}:${TAG} +docker-push-dsnexec: docker-build-dsnexec + docker push ${DSNEXEC_IMG_PATH}:${TAG} + .push-${TAG}: docker-build docker push ${IMG_PATH}:${TAG} @touch $@ From 0db8bdf223a21c9632d5349242afba5b66eb6e29 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Thu, 21 Sep 2023 21:58:39 +0530 Subject: [PATCH 3/5] added docker build and push for dsnexec in github actions --- .github/workflows/build-main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-main.yml b/.github/workflows/build-main.yml index 1d300554..39ad1fc1 100644 --- a/.github/workflows/build-main.yml +++ b/.github/workflows/build-main.yml @@ -32,6 +32,7 @@ jobs: echo 'db-controller-namespace' > .id make docker-build make docker-build-dbproxy + make docker-build-dsnexec - name: Push to GHCR env: @@ -39,6 +40,7 @@ jobs: run: | make docker-push make docker-push-dbproxy + make docker-push-dsnexec - uses: act10ns/slack@v1 with: status: ${{ job.status }} From 0cc00737dd47f5c3627b3ab8e9441025aae26411 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Thu, 21 Sep 2023 23:30:03 +0530 Subject: [PATCH 4/5] changed conflicting names of webhook client config services between dbproxy and dbdsn --- helm/db-controller/templates/webhook.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/db-controller/templates/webhook.yaml b/helm/db-controller/templates/webhook.yaml index d1ed3c11..0ecaa79d 100644 --- a/helm/db-controller/templates/webhook.yaml +++ b/helm/db-controller/templates/webhook.yaml @@ -34,7 +34,7 @@ webhooks: - clientConfig: caBundle: Cg== service: - name: {{ include "db-controller.fullname" . }} + name: {{ include "db-controller.fullname" . }}-dsnexec path: /mutate-dsnexec port: 7443 namespace: {{ .Release.Namespace }} From c0331bb75fcff96a3d88bd7397b4c43e2e0dfd30 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Fri, 22 Sep 2023 11:52:44 +0530 Subject: [PATCH 5/5] review comments incorporated - skipping webhook server creation if not necessory --- cmd/manager/main.go | 81 +++++++++++++++++----------------- helm/db-controller/values.yaml | 2 +- 2 files changed, 41 insertions(+), 42 deletions(-) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 08c84533..ca3e2459 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -173,49 +173,48 @@ func main() { os.Exit(1) } - webHookServer := mgr.GetWebhookServer() - - webHookServer.Port = 7443 - webHookServer.CertDir = "./certs/" - - if enableDBProxyWebhook { - - cfg, err := dbwebhook.ParseConfig(dBProxySidecarConfigPath) - - if err != nil { - setupLog.Error(err, "could not parse db proxy sidecar configuration") - os.Exit(1) + if enableDBProxyWebhook || enableDsnExecWebhook { + webHookServer := mgr.GetWebhookServer() + webHookServer.Port = 7443 + webHookServer.CertDir = "./certs/" + if enableDBProxyWebhook { + + cfg, err := dbwebhook.ParseConfig(dBProxySidecarConfigPath) + + if err != nil { + setupLog.Error(err, "could not parse db proxy sidecar configuration") + os.Exit(1) + } + setupLog.Info("Parsed db proxy conig:", "dbproxysidecarconfig", cfg) + + setupLog.Info("registering with webhook server for DbProxy") + webHookServer.Register("/mutate", &webhook.Admission{ + Handler: &dbwebhook.DBProxyInjector{ + Name: "DB Proxy", + Client: mgr.GetClient(), + DBProxySidecarConfig: cfg, + }, + }) } - setupLog.Info("Parsed db proxy conig:", "dbproxysidecarconfig", cfg) - - setupLog.Info("registering with webhook server for DbProxy") - webHookServer.Register("/mutate", &webhook.Admission{ - Handler: &dbwebhook.DBProxyInjector{ - Name: "DB Proxy", - Client: mgr.GetClient(), - DBProxySidecarConfig: cfg, - }, - }) - } - - if enableDsnExecWebhook { - - cfg, err := dbwebhook.ParseConfig(dsnExecSidecarConfigPath) - - if err != nil { - setupLog.Error(err, "could not parse dsnexec sidecar configuration") - os.Exit(1) + if enableDsnExecWebhook { + + cfg, err := dbwebhook.ParseConfig(dsnExecSidecarConfigPath) + + if err != nil { + setupLog.Error(err, "could not parse dsnexec sidecar configuration") + os.Exit(1) + } + setupLog.Info("Parsed dsnexec conig:", "dsnexecsidecarconfig", cfg) + + setupLog.Info("registering with webhook server for DsnExec") + webHookServer.Register("/mutate-dsnexec", &webhook.Admission{ + Handler: &dbwebhook.DsnExecInjector{ + Name: "Dsnexec", + Client: mgr.GetClient(), + DsnExecSidecarConfig: cfg, + }, + }) } - setupLog.Info("Parsed dsnexec conig:", "dsnexecsidecarconfig", cfg) - - setupLog.Info("registering with webhook server for DsnExec") - webHookServer.Register("/mutate-dsnexec", &webhook.Admission{ - Handler: &dbwebhook.DsnExecInjector{ - Name: "Dsnexec", - Client: mgr.GetClient(), - DsnExecSidecarConfig: cfg, - }, - }) } setupLog.Info("starting manager") diff --git a/helm/db-controller/values.yaml b/helm/db-controller/values.yaml index 163b4848..450d8431 100644 --- a/helm/db-controller/values.yaml +++ b/helm/db-controller/values.yaml @@ -109,7 +109,7 @@ dbproxy: dsnexec: enabled: true image: - repository: "" + repository: "ghcr.io/infobloxopen/dsnexec" tag: "" zapLogger: