From 180bd7e55bcf8e5369e2f20933575951f59f986d Mon Sep 17 00:00:00 2001 From: Sergio Rua Date: Fri, 2 Aug 2024 16:36:34 +0100 Subject: [PATCH] Limit namespaces scope --- charts/axonops-developer-operator/Chart.yaml | 4 +- .../templates/deployment.yaml | 12 ++++ .../templates/serviceaccount.yaml | 16 ++++- charts/axonops-developer-operator/values.yaml | 10 +++ cmd/main.go | 44 +++++++++--- .../controller/axonopscassandra_controller.go | 69 +++++++++++++++---- 6 files changed, 127 insertions(+), 28 deletions(-) diff --git a/charts/axonops-developer-operator/Chart.yaml b/charts/axonops-developer-operator/Chart.yaml index c9e9e0e..403bf31 100644 --- a/charts/axonops-developer-operator/Chart.yaml +++ b/charts/axonops-developer-operator/Chart.yaml @@ -11,10 +11,10 @@ kubeVersion: ">= 1.24.0-0" type: application # Chart version -version: 0.1.0 +version: 0.2.0 # Latest container tag -appVersion: v0.1.0 +appVersion: v0.1.0-beta1 maintainers: - email: info@axonops.com diff --git a/charts/axonops-developer-operator/templates/deployment.yaml b/charts/axonops-developer-operator/templates/deployment.yaml index 350b6fd..80f4912 100644 --- a/charts/axonops-developer-operator/templates/deployment.yaml +++ b/charts/axonops-developer-operator/templates/deployment.yaml @@ -57,6 +57,18 @@ spec: - containerPort: {{ .Values.metricsPort | default 8080 }} name: metrics protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 3 + periodSeconds: 3 + readinessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 3 + periodSeconds: 3 {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/axonops-developer-operator/templates/serviceaccount.yaml b/charts/axonops-developer-operator/templates/serviceaccount.yaml index 683098a..fa8a7a7 100644 --- a/charts/axonops-developer-operator/templates/serviceaccount.yaml +++ b/charts/axonops-developer-operator/templates/serviceaccount.yaml @@ -10,7 +10,6 @@ rules: - "networking" resources: - "ingresses" - - "services" verbs: - "get" - "list" @@ -21,7 +20,18 @@ rules: - apiGroups: - "apps" resources: - - "ingresses" + - "deployments" + - "statefulsets" + verbs: + - "get" + - "list" + - "watch" + - "update" + - "delete" + - "create" +- apiGroups: + - "" + resources: - "services" verbs: - "get" @@ -40,7 +50,7 @@ rules: - apiGroups: - "axonops.com" resources: - - "axonopscassandra" + - "axonopscassandras" verbs: - "get" - "list" diff --git a/charts/axonops-developer-operator/values.yaml b/charts/axonops-developer-operator/values.yaml index e3f6d05..3693e31 100644 --- a/charts/axonops-developer-operator/values.yaml +++ b/charts/axonops-developer-operator/values.yaml @@ -27,6 +27,16 @@ enableDbSecrets: true # additional arguments to operator args: [] + # - metrics-bind-address=:8080 + # - health-probe-bind-address=:8081 + # - leader-elect=true + # - leader-election-id=axonops-developer-operator + # - watch-namespaces=default,one,two + +# additional environment variables to operator +env: [] + # - name: MY_ENV_VAR + # value: "my value" environmentSecret: "" diff --git a/cmd/main.go b/cmd/main.go index 2b3333b..0047243 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "flag" "os" + "strings" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -30,6 +31,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -58,6 +60,7 @@ func main() { var probeAddr string var secureMetrics bool var enableHTTP2 bool + var watchNamespaces string flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metric endpoint binds to. "+ "Use the port :8080. If not set, it will be '0 in order to disable the metrics server") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -68,6 +71,7 @@ func main() { "If set the metrics endpoint is served securely") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") + flag.StringVar(&watchNamespaces, "watch-namespaces", "", "Comma separated list of namespaces that vals-operator will watch.") opts := zap.Options{ Development: true, } @@ -96,6 +100,34 @@ func main() { TLSOpts: tlsOpts, }) + var cacheOptions cache.Options + + if watchNamespaces != "" { + setupLog.Info("watching namespaces", "namespaces", watchNamespaces) + + // Split the watchNamespaces string into a slice of namespaces + namespaces := strings.Split(watchNamespaces, ",") + + // Create a map to hold namespace configurations + namespaceConfigs := make(map[string]cache.Config) + + // Add each namespace to the map + for _, ns := range namespaces { + // Trim any whitespace from the namespace + ns = strings.TrimSpace(ns) + if ns != "" { + namespaceConfigs[ns] = cache.Config{} + } + } + + // Set the cache options with the namespace configurations + cacheOptions = cache.Options{ + DefaultNamespaces: namespaceConfigs, + } + + setupLog.Info("configured cache for namespaces", "count", len(namespaceConfigs)) + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{ @@ -107,17 +139,7 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "c9da0915.axonops.com", - // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily - // when the Manager ends. This requires the binary to immediately end when the - // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly - // speeds up voluntary leader transitions as the new leader don't have to wait - // LeaseDuration time first. - // - // In the default scaffold provided, the program ends immediately after - // the manager stops, so would be fine to enable this option. However, - // if you are doing or is intended to do any operation such as perform cleanups - // after the manager stops then its usage might be unsafe. - // LeaderElectionReleaseOnCancel: true, + Cache: cacheOptions, }) if err != nil { setupLog.Error(err, "unable to start manager") diff --git a/internal/controller/axonopscassandra_controller.go b/internal/controller/axonopscassandra_controller.go index 74e83f0..0c92b1b 100644 --- a/internal/controller/axonopscassandra_controller.go +++ b/internal/controller/axonopscassandra_controller.go @@ -23,6 +23,7 @@ import ( "github.com/axonops/axonops-developer-operator/apps" "github.com/axonops/axonops-developer-operator/utils" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -39,6 +40,7 @@ import ( type AxonOpsCassandraReconciler struct { client.Client ReconciliationPeriod time.Duration + Recorder record.EventRecorder Scheme *runtime.Scheme Ctx context.Context } @@ -134,24 +136,34 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req */ var elasticStatefulSet *appsv1.StatefulSet - elasticStatefulSet, err = r.getSts("es-"+thisClusterName, thisClusterNamespace) + var elasticCurrentStatefulSet *appsv1.StatefulSet + elasticCurrentStatefulSet, err = r.getSts("es-"+thisClusterName, thisClusterNamespace) if client.IgnoreNotFound(err) != nil { return ctrl.Result{}, err } + /* Create the elastic search STS */ + elasticStatefulSet, err = apps.GenerateElasticsearchConfig(axonopsCassCluster) + if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to parse the Elasticsearch config: "+err.Error()) + return ctrl.Result{}, err + } - if elasticStatefulSet == nil { - /* Create the elastic search STS */ - elasticStatefulSet, err = apps.GenerateElasticsearchConfig(axonopsCassCluster) + if elasticCurrentStatefulSet == nil { + err = r.Create(ctx, elasticStatefulSet) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the Elasticsearch sts: "+err.Error()) return ctrl.Result{}, err } - - err = r.Create(ctx, elasticStatefulSet) + } else { + /* Update the elastic search STS */ + err = r.Update(ctx, elasticStatefulSet) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to update the Elasticsearch sts: "+err.Error()) return ctrl.Result{}, err } } + var elasticSvc *corev1.Service elasticSvc, err = r.getService("es-"+thisClusterName, thisClusterNamespace) if client.IgnoreNotFound(err) != nil { @@ -161,6 +173,7 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req /* Create the elastic search service */ elasticSvc, err = apps.GenerateElasticsearchServiceConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to parse the Elasticsearch service config: "+err.Error()) return ctrl.Result{}, err } @@ -176,24 +189,34 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req */ var dashDeployment *appsv1.Deployment - dashDeployment, err = r.getDeployment("ds-"+thisClusterName, thisClusterNamespace) + var dashDeploymentCurrent *appsv1.Deployment + dashDeploymentCurrent, err = r.getDeployment("ds-"+thisClusterName, thisClusterNamespace) if client.IgnoreNotFound(err) != nil { return ctrl.Result{}, err } - if dashDeployment == nil { - /* Create the dash search STS */ - dashDeployment, err = apps.GenerateDashboardConfig(axonopsCassCluster) + /* Create the dash search STS */ + dashDeployment, err = apps.GenerateDashboardConfig(axonopsCassCluster) + if err != nil { + return ctrl.Result{}, err + } + + if dashDeploymentCurrent == nil { + err = r.Create(ctx, dashDeployment) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the AxonOps dashboard: "+err.Error()) return ctrl.Result{}, err } - - err = r.Create(ctx, dashDeployment) + } else { + /* Update the dash search STS */ + err = r.Update(ctx, dashDeployment) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to update the AxonOps dashboard: "+err.Error()) return ctrl.Result{}, err } } + var dashSvc *corev1.Service dashSvc, err = r.getService("ds-"+thisClusterName, thisClusterNamespace) if client.IgnoreNotFound(err) != nil { @@ -203,11 +226,13 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req /* Create the dash search service */ dashSvc, err = apps.GenerateDashboardServiceConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to parse the AxonOps dashboard config: "+err.Error()) return ctrl.Result{}, err } err = r.Create(ctx, dashSvc) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the AxonOps service: "+err.Error()) return ctrl.Result{}, err } } @@ -221,18 +246,21 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req } dashIngress, err = apps.GenerateDashboardIngressConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Could not parse the AxonOps ingress: "+err.Error()) return ctrl.Result{}, err } if dashIngressCurrent == nil { err = r.Create(ctx, dashIngress) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the AxonOps ingress: "+err.Error()) return ctrl.Result{}, err } } else { /* Update the dash search Ingress */ err = r.Update(ctx, dashIngress) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to update the AxonOps ingress: "+err.Error()) return ctrl.Result{}, err } } @@ -253,11 +281,13 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req /* Create the axonServer search STS */ axonServerSts, err = apps.GenerateServerConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to parse the AxonOps configuration: "+err.Error()) return ctrl.Result{}, err } if axonServerStsCurrent == nil { err = r.Create(ctx, axonServerSts) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to created the AxonOps service: "+err.Error()) return ctrl.Result{}, err } } else { @@ -277,17 +307,20 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req /* Create the axonServer search service */ axonServerSvc, err = apps.GenerateServerServiceConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to parse the AxonOps configuration: "+err.Error()) return ctrl.Result{}, err } if axonServerSvcCurrent == nil { err = r.Create(ctx, axonServerSvc) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the AxonOps service: "+err.Error()) return ctrl.Result{}, err } } else { /* Update the axonServer search service */ err = r.Update(ctx, axonServerSvc) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to update the AxonOps service: "+err.Error()) return ctrl.Result{}, err } } @@ -308,20 +341,25 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req /* Create the cassandra search STS */ cassandraStatefulSet, err = apps.GenerateCassandraConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to parse the Cassandra configuration: "+err.Error()) return ctrl.Result{}, err } if cassandraStatefulSetCurrent == nil { err = r.Create(ctx, cassandraStatefulSet) if err != nil { + //r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the Cassandra Statefulset: "+err.Error()) return ctrl.Result{}, err } + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Created", "Cassandra sts created successfully") } else { /* Update the cassandra search STS */ err = r.Update(ctx, cassandraStatefulSet) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to update the Cassandra Statefulset: "+err.Error()) return ctrl.Result{}, err } + //r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Updated", "Cassandra sts updated successfully") } var cassandraSvc *corev1.Service @@ -333,6 +371,7 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req /* Create the cassandra search service */ cassandraSvc, err = apps.GenerateCassandraServiceConfig(axonopsCassCluster) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to create the Cassandra service: "+err.Error()) return ctrl.Result{}, err } if cassandraSvcCurrent == nil { @@ -340,14 +379,19 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req if err != nil { return ctrl.Result{}, err } + //r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Created", "Cassandra service created successfully") } else { /* Update the cassandra search service */ err = r.Update(ctx, cassandraSvc) if err != nil { + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Failed", "Failed to update the Cassandra service: "+err.Error()) return ctrl.Result{}, err } + //r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Created", "Cassandra service updated successfully") } + r.Recorder.Event(&axonopsCassCluster, corev1.EventTypeNormal, "Created", "Environment created successfully") + // condition := metav1.Condition{ // Type: "Ready", // Status: metav1.ConditionTrue, @@ -372,6 +416,7 @@ func (r *AxonOpsCassandraReconciler) Reconcile(ctx context.Context, req ctrl.Req // SetupWithManager sets up the controller with the Manager. func (r *AxonOpsCassandraReconciler) SetupWithManager(mgr ctrl.Manager) error { pred := predicate.GenerationChangedPredicate{} + r.Recorder = mgr.GetEventRecorderFor("AxonDev") return ctrl.NewControllerManagedBy(mgr). For(&cassandraaxonopscomv1beta1.AxonOpsCassandra{}).