From 7e614bbcb36bdf65133ebb90ee8d616d2915bdce Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Mon, 15 Jan 2024 20:23:21 -0800 Subject: [PATCH] Dynamically start prometheus and service monitor Signed-off-by: Tamal Saha --- Makefile | 2 +- go.mod | 4 +- go.sum | 8 +- pkg/apiserver/apiserver.go | 67 +++++++++----- pkg/registry/ui/dashboardgroup/storage.go | 5 ++ .../client-go/apiextensions/controller.go | 87 +++++++++++++++++++ .../client-go/apiextensions/kubernetes.go | 20 +++++ vendor/modules.txt | 4 +- 8 files changed, 167 insertions(+), 30 deletions(-) create mode 100644 vendor/kmodules.xyz/client-go/apiextensions/controller.go diff --git a/Makefile b/Makefile index 975786c2b..d113bf152 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ lint: $(BUILD_DIRS) --env GO111MODULE=on \ --env GOFLAGS="-mod=vendor" \ $(BUILD_IMAGE) \ - golangci-lint run --enable $(ADDTL_LINTERS) --timeout=10m --skip-files="generated.*\.go$\" --skip-dirs-use-default --skip-dirs=client,vendor + golangci-lint run --enable $(ADDTL_LINTERS) --max-same-issues=100 --timeout=10m --skip-files="generated.*\.go$\" --skip-dirs-use-default --skip-dirs=client,vendor $(BUILD_DIRS): @mkdir -p $@ diff --git a/go.mod b/go.mod index a7f522452..230de10c1 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/zeebo/xxh3 v1.0.2 go.bytebuilders.dev/license-verifier v0.13.4 - go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb + go.openviz.dev/apimachinery v0.0.7 go.openviz.dev/grafana-sdk v0.0.4 golang.org/x/text v0.14.0 gomodules.xyz/logs v0.0.7 @@ -28,7 +28,7 @@ require ( k8s.io/client-go v0.29.0 k8s.io/klog/v2 v2.110.1 kmodules.xyz/authorizer v0.29.0 - kmodules.xyz/client-go v0.29.5 + kmodules.xyz/client-go v0.29.6 kmodules.xyz/custom-resources v0.29.0 kmodules.xyz/monitoring-agent-api v0.29.0 sigs.k8s.io/controller-runtime v0.16.3 diff --git a/go.sum b/go.sum index 8b13a4ab9..addc7efe4 100644 --- a/go.sum +++ b/go.sum @@ -473,8 +473,8 @@ go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= -go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb h1:1wDqFOBePci3nBcUEMxIa37p/hkSVZczfCvUJjCNHnw= -go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb/go.mod h1:cWwPKNiOIOpXV4xwFNJQaeq2+pLKVwg5KhsiCiEDeB8= +go.openviz.dev/apimachinery v0.0.7 h1:LBW/RtI21FA9ILHEfOMCF+YCSBw2UEHd8H+cgmtZaAs= +go.openviz.dev/apimachinery v0.0.7/go.mod h1:OUaaMC3iCOyp7R6j5puYMV7fkSNu6ERW2P+jcJ+n1iE= go.openviz.dev/grafana-sdk v0.0.4 h1:Tl/lZtHK22j5eGWBQpdi2ErqS5TFlZeICCUyZnIJx1g= go.openviz.dev/grafana-sdk v0.0.4/go.mod h1:fdBkehPrBOKL53+fCCm95AzK5Vs/5+xhsfuXo8OblaY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= @@ -759,8 +759,8 @@ kmodules.xyz/apiversion v0.2.0 h1:vAQYqZFm4xu4pbB1cAdHbFEPES6EQkcR4wc06xdTOWk= kmodules.xyz/apiversion v0.2.0/go.mod h1:oPX8g8LvlPdPX3Yc5YvCzJHQnw3YF/X4/jdW0b1am80= kmodules.xyz/authorizer v0.29.0 h1:ND8YGeyzExdZ8Bq5Z6UdFO794I6+oPuXbUMWyjlsYgM= kmodules.xyz/authorizer v0.29.0/go.mod h1:UQmE3sNXeliebUqjEeD9QYiY+Na27/C5Bg/ekVRfQ3U= -kmodules.xyz/client-go v0.29.5 h1:iRl4MoV+96TM1csInOCWjn5xSOXzuYlil6CO40vXLHU= -kmodules.xyz/client-go v0.29.5/go.mod h1:pHuzpwzEcDUIGjVVvwz9N8lY+6A7HXwvs2d7NtK7Hho= +kmodules.xyz/client-go v0.29.6 h1:xTVq5LZvsPBUTLY7PORq7zveLOj/vpuTDvkpHWOk3RM= +kmodules.xyz/client-go v0.29.6/go.mod h1:pHuzpwzEcDUIGjVVvwz9N8lY+6A7HXwvs2d7NtK7Hho= kmodules.xyz/crd-schema-fuzz v0.29.1 h1:zJTlWYOrT5dsVVHW8HGcnR/vaWfxQfNh11QwTtkYpcs= kmodules.xyz/crd-schema-fuzz v0.29.1/go.mod h1:n708z9YQqLMP2KNLQVgBcRJw1QpSWLvpNCEi+KJDOYE= kmodules.xyz/custom-resources v0.29.0 h1:RaDM2+wSVXiwIvLqmkTVYpwoH83AC8wruXe2p2rOZNY= diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 033e0b901..b49b4b018 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -31,8 +31,10 @@ import ( servicemonitorcontroller "go.openviz.dev/grafana-tools/pkg/controllers/servicemonitor" dashgroupstorage "go.openviz.dev/grafana-tools/pkg/registry/ui/dashboardgroup" + "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" core "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -46,9 +48,11 @@ import ( "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" "kmodules.xyz/authorizer" + "kmodules.xyz/client-go/apiextensions" clustermeta "kmodules.xyz/client-go/cluster" appcatalogapi "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1" mona "kmodules.xyz/monitoring-agent-api/api/v1" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -71,6 +75,7 @@ func init() { utilruntime.Must(monitoringv1.AddToScheme(Scheme)) utilruntime.Must(appcatalogapi.AddToScheme(Scheme)) utilruntime.Must(chartsapi.AddToScheme(Scheme)) + utilruntime.Must(apiextensionsv1.AddToScheme(Scheme)) // we need to add the options to empty v1 // TODO fix the server code to avoid this @@ -141,7 +146,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) { // ctrl.SetLogger(...) log.SetLogger(klogr.New()) // nolint:staticcheck - mgr, err := manager.New(c.ExtraConfig.ClientConfig, manager.Options{ + mgr, err := ctrl.NewManager(c.ExtraConfig.ClientConfig, ctrl.Options{ Scheme: Scheme, Metrics: metricsserver.Options{BindAddress: ""}, HealthProbeBindAddress: "", @@ -173,30 +178,50 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) { } } - if err = promtehsucontroller.NewReconciler( - mgr.GetClient(), - bc, - cid, - ).SetupWithManager(mgr); err != nil { - klog.Error(err, "unable to create controller", "controller", "Prometheus") + if err := apiextensions.NewReconciler(ctx, mgr).SetupWithManager(mgr); err != nil { + klog.Error(err, "unable to create controller controller CustomResourceReconciler") os.Exit(1) } - if err = servicemonitorcontroller.NewFederationReconciler( - c.ExtraConfig.ClientConfig, - mgr.GetClient(), - ).SetupWithManager(mgr); err != nil { - klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor") - os.Exit(1) - } + apiextensions.RegisterSetup(schema.GroupKind{ + Group: monitoring.GroupName, + Kind: monitoringv1.PrometheusesKind, + }, func(ctx context.Context, mgr ctrl.Manager) { + if err = promtehsucontroller.NewReconciler( + mgr.GetClient(), + bc, + cid, + ).SetupWithManager(mgr); err != nil { + klog.Error(err, "unable to create controller", "controller", "Prometheus") + os.Exit(1) + } + }) - if err = servicemonitorcontroller.NewAutoReconciler( - c.ExtraConfig.ClientConfig, - mgr.GetClient(), - ).SetupWithManager(mgr); err != nil { - klog.Error(err, "unable to create controller", "auto controller", "ServiceMonitor") - os.Exit(1) - } + apiextensions.RegisterSetup(schema.GroupKind{ + Group: monitoring.GroupName, + Kind: monitoringv1.ServiceMonitorsKind, + }, func(ctx context.Context, mgr ctrl.Manager) { + if err = servicemonitorcontroller.NewFederationReconciler( + c.ExtraConfig.ClientConfig, + mgr.GetClient(), + ).SetupWithManager(mgr); err != nil { + klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor") + os.Exit(1) + } + }) + + apiextensions.RegisterSetup(schema.GroupKind{ + Group: monitoring.GroupName, + Kind: monitoringv1.ServiceMonitorsKind, + }, func(ctx context.Context, mgr ctrl.Manager) { + if err = servicemonitorcontroller.NewAutoReconciler( + c.ExtraConfig.ClientConfig, + mgr.GetClient(), + ).SetupWithManager(mgr); err != nil { + klog.Error(err, "unable to create controller", "auto controller", "ServiceMonitor") + os.Exit(1) + } + }) if err := mgr.GetFieldIndexer().IndexField(context.Background(), &appcatalogapi.AppBinding{}, mona.DefaultGrafanaKey, func(rawObj client.Object) []string { app := rawObj.(*appcatalogapi.AppBinding) diff --git a/pkg/registry/ui/dashboardgroup/storage.go b/pkg/registry/ui/dashboardgroup/storage.go index 14608042f..76954dc9a 100644 --- a/pkg/registry/ui/dashboardgroup/storage.go +++ b/pkg/registry/ui/dashboardgroup/storage.go @@ -56,6 +56,7 @@ var ( _ rest.Scoper = &Storage{} _ rest.Storage = &Storage{} _ rest.Creater = &Storage{} + _ rest.SingularNameProvider = &Storage{} ) func NewStorage(kc client.Client, a authorizer.Authorizer) *Storage { @@ -70,6 +71,10 @@ func (r *Storage) GroupVersionKind(_ schema.GroupVersion) schema.GroupVersionKin return uiapi.SchemeGroupVersion.WithKind(uiapi.ResourceKindDashboardGroup) } +func (r *Storage) GetSingularName() string { + return strings.ToLower(uiapi.ResourceKindDashboardGroup) +} + func (r *Storage) NamespaceScoped() bool { return false } diff --git a/vendor/kmodules.xyz/client-go/apiextensions/controller.go b/vendor/kmodules.xyz/client-go/apiextensions/controller.go new file mode 100644 index 000000000..94425010f --- /dev/null +++ b/vendor/kmodules.xyz/client-go/apiextensions/controller.go @@ -0,0 +1,87 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apiextensions + +import ( + "context" + "sync" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +type SetupFn func(ctx context.Context, mgr ctrl.Manager) + +var setupFns = map[schema.GroupKind]SetupFn{ + // schema.GroupKind{"compute.gcp.kubedb.com", "Firewall"}: firewall.Setup, +} + +var ( + setupDone = map[schema.GroupKind]bool{} + mu sync.Mutex +) + +type Reconciler struct { + ctx context.Context + mgr ctrl.Manager +} + +func NewReconciler(ctx context.Context, mgr ctrl.Manager) *Reconciler { + return &Reconciler{ctx: ctx, mgr: mgr} +} + +func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + var crd apiextensions.CustomResourceDefinition + if err := r.mgr.GetClient().Get(ctx, req.NamespacedName, &crd); err != nil { + log.Error(err, "unable to fetch CustomResourceDefinition") + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + gk := schema.GroupKind{ + Group: crd.Spec.Group, + Kind: crd.Spec.Names.Kind, + } + mu.Lock() + defer mu.Unlock() + _, found := setupDone[gk] + if found { + return ctrl.Result{}, nil + } + setup, found := setupFns[gk] + if found { + setup(r.ctx, r.mgr) + setupDone[gk] = true + } + return ctrl.Result{}, nil +} + +func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&apiextensions.CustomResourceDefinition{}). + Complete(r) +} + +func RegisterSetup(gk schema.GroupKind, fn SetupFn) { + mu.Lock() + defer mu.Unlock() + + setupFns[gk] = fn +} diff --git a/vendor/kmodules.xyz/client-go/apiextensions/kubernetes.go b/vendor/kmodules.xyz/client-go/apiextensions/kubernetes.go index cab174972..63052359d 100644 --- a/vendor/kmodules.xyz/client-go/apiextensions/kubernetes.go +++ b/vendor/kmodules.xyz/client-go/apiextensions/kubernetes.go @@ -109,3 +109,23 @@ func WaitForCRDReady(client crd_cs.Interface, crds []*CustomResourceDefinition) }) return errors.Wrap(err, "timed out waiting for CRD") } + +func RemoveCRDs(client crd_cs.Interface, crds []*CustomResourceDefinition) error { + for _, crd := range crds { + // Use crd v1 for k8s >= 1.16, if available + // ref: https://github.com/kubernetes/kubernetes/issues/91395 + if crd.V1 == nil { + gvr := schema.GroupVersionResource{ + Group: crd.V1beta1.Spec.Group, + Version: crd.V1beta1.Spec.Versions[0].Name, + Resource: crd.V1beta1.Spec.Names.Plural, + } + return fmt.Errorf("missing V1 definition for %s", gvr) + } + err := client.ApiextensionsV1().CustomResourceDefinitions().Delete(context.TODO(), crd.V1.Name, metav1.DeleteOptions{}) + if err != nil && !kerr.IsNotFound(err) { + return err + } + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7a185a57a..aa9001731 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -408,7 +408,7 @@ go.opentelemetry.io/proto/otlp/collector/trace/v1 go.opentelemetry.io/proto/otlp/common/v1 go.opentelemetry.io/proto/otlp/resource/v1 go.opentelemetry.io/proto/otlp/trace/v1 -# go.openviz.dev/apimachinery v0.0.7-0.20240105050246-85dfcd1152cb +# go.openviz.dev/apimachinery v0.0.7 ## explicit; go 1.21.5 go.openviz.dev/apimachinery/apis/openviz go.openviz.dev/apimachinery/apis/openviz/install @@ -1325,7 +1325,7 @@ kmodules.xyz/authorizer/apiserver kmodules.xyz/authorizer/rbac kmodules.xyz/authorizer/rbac/helpers kmodules.xyz/authorizer/rbac/validation -# kmodules.xyz/client-go v0.29.5 +# kmodules.xyz/client-go v0.29.6 ## explicit; go 1.21.5 kmodules.xyz/client-go kmodules.xyz/client-go/api/v1