diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 4aa3351fa..e42f37672 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -29,6 +29,7 @@ import ( uiapi "go.openviz.dev/apimachinery/apis/ui/v1alpha1" promtehsucontroller "go.openviz.dev/grafana-tools/pkg/controllers/prometheus" servicemonitorcontroller "go.openviz.dev/grafana-tools/pkg/controllers/servicemonitor" + "go.openviz.dev/grafana-tools/pkg/detector" dashgroupstorage "go.openviz.dev/grafana-tools/pkg/registry/ui/dashboardgroup" "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring" @@ -183,6 +184,8 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) { os.Exit(1) } + d := detector.New(mgr.GetClient()) + apiextensions.RegisterSetup(schema.GroupKind{ Group: monitoring.GroupName, Kind: monitoringv1.PrometheusesKind, @@ -191,6 +194,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) { mgr.GetClient(), bc, cid, + d, ).SetupWithManager(mgr); err != nil { klog.Error(err, "unable to create controller", "controller", "Prometheus") os.Exit(1) @@ -212,6 +216,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) { if err = servicemonitorcontroller.NewFederationReconciler( c.ExtraConfig.ClientConfig, mgr.GetClient(), + d, ).SetupWithManager(mgr); err != nil { klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor") os.Exit(1) diff --git a/pkg/controllers/prometheus/prometheus_controller.go b/pkg/controllers/prometheus/prometheus_controller.go index df743be50..e5e92bd17 100644 --- a/pkg/controllers/prometheus/prometheus_controller.go +++ b/pkg/controllers/prometheus/prometheus_controller.go @@ -21,6 +21,7 @@ import ( "fmt" openvizapi "go.openviz.dev/apimachinery/apis/openviz/v1alpha1" + "go.openviz.dev/grafana-tools/pkg/detector" "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -53,14 +54,16 @@ type PrometheusReconciler struct { scheme *runtime.Scheme bc *Client clusterUID string + d detector.Detector } -func NewReconciler(kc client.Client, bc *Client, clusterUID string) *PrometheusReconciler { +func NewReconciler(kc client.Client, bc *Client, clusterUID string, d detector.Detector) *PrometheusReconciler { return &PrometheusReconciler{ kc: kc, scheme: kc.Scheme(), bc: bc, clusterUID: clusterUID, + d: d, } } @@ -85,21 +88,15 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, client.IgnoreNotFound(err) } - cm := clustermeta.DetectClusterManager(r.kc) - gvk := schema.GroupVersionKind{ - Group: monitoring.GroupName, - Version: monitoringv1.Version, - Kind: "Prometheus", - } - - key := client.ObjectKeyFromObject(&prom) - isDefault, err := r.IsDefault(cm, gvk, key) - if err != nil { + if ready, err := r.d.Ready(); !ready { return ctrl.Result{}, err } + key := req.NamespacedName + isDefault := r.d.IsDefault(key) + if prom.DeletionTimestamp != nil { - err := r.CleanupPreset(cm, &prom, isDefault) + err := r.CleanupPreset(&prom, isDefault) if err != nil { return ctrl.Result{}, err } @@ -146,7 +143,7 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request) } klog.Infof("%s Prometheus %s/%s to add finalizer %s", vt, prom.Namespace, prom.Name, mona.PrometheusKey) - if err := r.SetupClusterForPrometheus(cm, &prom, isDefault); err != nil { + if err := r.SetupClusterForPrometheus(&prom, isDefault); err != nil { log.Error(err, "unable to setup Prometheus") return ctrl.Result{}, err } @@ -154,14 +151,6 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } -func (r *PrometheusReconciler) IsDefault(cm kmapi.ClusterManager, gvk schema.GroupVersionKind, key types.NamespacedName) (bool, error) { - if cm.ManagedByRancher() { - return key.Namespace == clustermeta.NamespaceRancherMonitoring && - key.Name == clustermeta.PrometheusRancherMonitoring, nil - } - return clustermeta.IsSingletonResource(r.kc, gvk, key) -} - func (r *PrometheusReconciler) findServiceForPrometheus(key types.NamespacedName) (*core.Service, error) { var svc core.Service err := r.kc.Get(context.TODO(), key, &svc) @@ -176,7 +165,7 @@ const ( saTrickster = "trickster" ) -func (r *PrometheusReconciler) SetupClusterForPrometheus(cm kmapi.ClusterManager, prom *monitoringv1.Prometheus, isDefault bool) error { +func (r *PrometheusReconciler) SetupClusterForPrometheus(prom *monitoringv1.Prometheus, isDefault bool) error { key := client.ObjectKeyFromObject(prom) svc, err := r.findServiceForPrometheus(key) @@ -273,7 +262,7 @@ func (r *PrometheusReconciler) SetupClusterForPrometheus(cm kmapi.ClusterManager } klog.Infof("%s role binding %s/%s", rbvt, rb.Namespace, rb.Name) - err = r.CreatePreset(cm, prom, isDefault) + err = r.CreatePreset(prom, isDefault) if err != nil { return err } @@ -362,21 +351,21 @@ var defaultPresetsLabels = map[string]string{ "charts.x-helm.dev/is-default-preset": "true", } -func (r *PrometheusReconciler) CreatePreset(cm kmapi.ClusterManager, p *monitoringv1.Prometheus, isDefault bool) error { +func (r *PrometheusReconciler) CreatePreset(p *monitoringv1.Prometheus, isDefault bool) error { presets := r.GeneratePresetForPrometheus(*p) presetBytes, err := json.Marshal(presets) if err != nil { return err } - if cm.ManagedByRancher() && !isDefault { + if r.d.Federated() && !isDefault { return r.CreateProjectPreset(p, presetBytes) } return r.CreateClusterPreset(presetBytes) } -func (r *PrometheusReconciler) CleanupPreset(cm kmapi.ClusterManager, p *monitoringv1.Prometheus, isDefault bool) error { - if cm.ManagedByRancher() && !isDefault { +func (r *PrometheusReconciler) CleanupPreset(p *monitoringv1.Prometheus, isDefault bool) error { + if r.d.Federated() && !isDefault { ns, _, err := r.NamespaceForProjectSettings(p) if err != nil { return err diff --git a/pkg/controllers/servicemonitor/federation_controller.go b/pkg/controllers/servicemonitor/federation_controller.go index 9c59e4515..d2b76a4fd 100644 --- a/pkg/controllers/servicemonitor/federation_controller.go +++ b/pkg/controllers/servicemonitor/federation_controller.go @@ -23,6 +23,8 @@ import ( "sort" "strings" + "go.openviz.dev/grafana-tools/pkg/detector" + "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" core "k8s.io/api/core/v1" @@ -52,13 +54,15 @@ type FederationReconciler struct { cfg *rest.Config kc client.Client scheme *runtime.Scheme + d detector.Detector } -func NewFederationReconciler(cfg *rest.Config, kc client.Client) *FederationReconciler { +func NewFederationReconciler(cfg *rest.Config, kc client.Client, d detector.Detector) *FederationReconciler { return &FederationReconciler{ cfg: cfg, kc: kc, scheme: kc.Scheme(), + d: d, } } @@ -105,7 +109,11 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request) return prometheuses[i].Name < prometheuses[j].Name }) - if !clustermeta.IsRancherManaged(r.kc.RESTMapper()) { + if ready, err := r.d.Ready(); !ready { + return ctrl.Result{}, err + } + + if !r.d.Federated() { // non rancher err := r.updateServiceMonitorLabels(prometheuses[0], &svcMon) if err != nil { @@ -158,7 +166,7 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request) var errList []error for _, prom := range promList.Items { - isDefault := IsDefaultPrometheus(prom) + isDefault := r.d.IsDefault(client.ObjectKeyFromObject(prom)) if !isDefault && prom.Namespace == req.Namespace { err := fmt.Errorf("federated service monitor can't be in the same namespace with project Prometheus %s/%s", prom.Namespace, prom.Namespace) @@ -202,15 +210,6 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, errors.NewAggregate(errList) } -func IsDefaultPrometheus(prom *monitoringv1.Prometheus) bool { - expected := client.ObjectKey{ - Namespace: clustermeta.NamespaceRancherMonitoring, - Name: clustermeta.PrometheusRancherMonitoring, - } - pk := client.ObjectKeyFromObject(prom) - return pk == expected -} - func (r *FederationReconciler) updateServiceMonitorLabels(prom *monitoringv1.Prometheus, src *monitoringv1.ServiceMonitor) error { vt, err := cu.CreateOrPatch(context.TODO(), r.kc, src, func(in client.Object, createOp bool) client.Object { obj := in.(*monitoringv1.ServiceMonitor) diff --git a/pkg/detector/api.go b/pkg/detector/api.go new file mode 100644 index 000000000..89e21d048 --- /dev/null +++ b/pkg/detector/api.go @@ -0,0 +1,169 @@ +/* +Copyright AppsCode Inc. and Contributors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package detector + +import ( + "context" + "errors" + "sync" + + "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + clustermeta "kmodules.xyz/client-go/cluster" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var GVKPrometheus = schema.GroupVersionKind{ + Group: monitoring.GroupName, + Version: monitoringv1.Version, + Kind: "Prometheus", +} + +type Detector interface { + Ready() (bool, error) + RancherManaged_() bool + Federated() bool + IsDefault(key types.NamespacedName) bool +} + +func New(kc client.Client) Detector { + return &lazy{kc: kc} +} + +var errUnknown = errors.New("unknown") + +type lazy struct { + kc client.Client + delegated Detector + mu sync.Mutex +} + +var _ Detector = &lazy{} + +func (l *lazy) detect() error { + var list unstructured.UnstructuredList + list.SetGroupVersionKind(GVKPrometheus) + err := l.kc.List(context.TODO(), &list) + if err != nil { + return err + } + if len(list.Items) == 0 { + return errUnknown + } + + if clustermeta.IsRancherManaged(l.kc.RESTMapper()) { + for _, obj := range list.Items { + if obj.GetNamespace() == clustermeta.NamespaceRancherMonitoring && + obj.GetName() == clustermeta.PrometheusRancherMonitoring { + l.delegated = &federated{} // rancher style federated + return nil + } + } + + // rancher cluster but using prometheus directly + l.delegated = &standalone{rancher: true, singleton: len(list.Items) == 1} + return nil + } + + // using prometheus directly and not rancher managed + l.delegated = &standalone{rancher: false, singleton: len(list.Items) == 1} + return nil +} + +func (l *lazy) Ready() (bool, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.delegated == nil { + err := l.detect() + return err == nil, err + } + return true, nil +} + +func (l *lazy) RancherManaged_() bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.delegated == nil { + panic("NotReady") + } + return l.delegated.RancherManaged_() +} + +func (l *lazy) Federated() bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.delegated == nil { + panic("NotReady") + } + return l.delegated.Federated() +} + +func (l *lazy) IsDefault(key types.NamespacedName) bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.delegated == nil { + panic("NotReady") + } + return l.delegated.IsDefault(key) +} + +type federated struct{} + +var _ Detector = federated{} + +func (f federated) Ready() (bool, error) { + return true, nil +} + +func (f federated) RancherManaged_() bool { + return true +} + +func (f federated) Federated() bool { + return true +} + +func (f federated) IsDefault(key types.NamespacedName) bool { + return key.Namespace == clustermeta.NamespaceRancherMonitoring && + key.Name == clustermeta.PrometheusRancherMonitoring +} + +type standalone struct { + rancher bool + singleton bool +} + +var _ Detector = standalone{} + +func (s standalone) Ready() (bool, error) { + return true, nil +} + +func (s standalone) RancherManaged_() bool { + return s.rancher +} + +func (s standalone) Federated() bool { + return false +} + +func (s standalone) IsDefault(key types.NamespacedName) bool { + return s.singleton +}