Handle rancher cluster with kube-prometheus-stack deployment

pkg/apiserver/apiserver.go

@@ -29,6 +29,7 @@ import (
 	uiapi "go.openviz.dev/apimachinery/apis/ui/v1alpha1"
 	promtehsucontroller "go.openviz.dev/grafana-tools/pkg/controllers/prometheus"
 	servicemonitorcontroller "go.openviz.dev/grafana-tools/pkg/controllers/servicemonitor"
+	"go.openviz.dev/grafana-tools/pkg/detector"
 	dashgroupstorage "go.openviz.dev/grafana-tools/pkg/registry/ui/dashboardgroup"
 
 	"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
@@ -183,6 +184,8 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
 		os.Exit(1)
 	}
 
+	d := detector.New(mgr.GetClient())
+
 	apiextensions.RegisterSetup(schema.GroupKind{
 		Group: monitoring.GroupName,
 		Kind:  monitoringv1.PrometheusesKind,
@@ -191,6 +194,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
 			mgr.GetClient(),
 			bc,
 			cid,
+			d,
 		).SetupWithManager(mgr); err != nil {
 			klog.Error(err, "unable to create controller", "controller", "Prometheus")
 			os.Exit(1)
@@ -212,6 +216,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
 		if err = servicemonitorcontroller.NewFederationReconciler(
 			c.ExtraConfig.ClientConfig,
 			mgr.GetClient(),
+			d,
 		).SetupWithManager(mgr); err != nil {
 			klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor")
 			os.Exit(1)

pkg/controllers/prometheus/prometheus_controller.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	openvizapi "go.openviz.dev/apimachinery/apis/openviz/v1alpha1"
+	"go.openviz.dev/grafana-tools/pkg/detector"
 
 	"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
@@ -53,14 +54,16 @@ type PrometheusReconciler struct {
 	scheme     *runtime.Scheme
 	bc         *Client
 	clusterUID string
+	d          detector.Detector
 }
 
-func NewReconciler(kc client.Client, bc *Client, clusterUID string) *PrometheusReconciler {
+func NewReconciler(kc client.Client, bc *Client, clusterUID string, d detector.Detector) *PrometheusReconciler {
 	return &PrometheusReconciler{
 		kc:         kc,
 		scheme:     kc.Scheme(),
 		bc:         bc,
 		clusterUID: clusterUID,
+		d:          d,
 	}
 }
 
@@ -85,21 +88,15 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	cm := clustermeta.DetectClusterManager(r.kc)
-	gvk := schema.GroupVersionKind{
-		Group:   monitoring.GroupName,
-		Version: monitoringv1.Version,
-		Kind:    "Prometheus",
-	}
-
-	key := client.ObjectKeyFromObject(&prom)
-	isDefault, err := r.IsDefault(cm, gvk, key)
-	if err != nil {
+	if ready, err := r.d.Ready(); !ready {
 		return ctrl.Result{}, err
 	}
 
+	key := req.NamespacedName
+	isDefault := r.d.IsDefault(key)
+
 	if prom.DeletionTimestamp != nil {
-		err := r.CleanupPreset(cm, &prom, isDefault)
+		err := r.CleanupPreset(&prom, isDefault)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -146,22 +143,14 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 	klog.Infof("%s Prometheus %s/%s to add finalizer %s", vt, prom.Namespace, prom.Name, mona.PrometheusKey)
 
-	if err := r.SetupClusterForPrometheus(cm, &prom, isDefault); err != nil {
+	if err := r.SetupClusterForPrometheus(&prom, isDefault); err != nil {
 		log.Error(err, "unable to setup Prometheus")
 		return ctrl.Result{}, err
 	}
 
 	return ctrl.Result{}, nil
 }
 
-func (r *PrometheusReconciler) IsDefault(cm kmapi.ClusterManager, gvk schema.GroupVersionKind, key types.NamespacedName) (bool, error) {
-	if cm.ManagedByRancher() {
-		return key.Namespace == clustermeta.NamespaceRancherMonitoring &&
-			key.Name == clustermeta.PrometheusRancherMonitoring, nil
-	}
-	return clustermeta.IsSingletonResource(r.kc, gvk, key)
-}
-
 func (r *PrometheusReconciler) findServiceForPrometheus(key types.NamespacedName) (*core.Service, error) {
 	var svc core.Service
 	err := r.kc.Get(context.TODO(), key, &svc)
@@ -176,7 +165,7 @@ const (
 	saTrickster    = "trickster"
 )
 
-func (r *PrometheusReconciler) SetupClusterForPrometheus(cm kmapi.ClusterManager, prom *monitoringv1.Prometheus, isDefault bool) error {
+func (r *PrometheusReconciler) SetupClusterForPrometheus(prom *monitoringv1.Prometheus, isDefault bool) error {
 	key := client.ObjectKeyFromObject(prom)
 
 	svc, err := r.findServiceForPrometheus(key)
@@ -273,7 +262,7 @@ func (r *PrometheusReconciler) SetupClusterForPrometheus(cm kmapi.ClusterManager
 	}
 	klog.Infof("%s role binding %s/%s", rbvt, rb.Namespace, rb.Name)
 
-	err = r.CreatePreset(cm, prom, isDefault)
+	err = r.CreatePreset(prom, isDefault)
 	if err != nil {
 		return err
 	}
@@ -362,21 +351,21 @@ var defaultPresetsLabels = map[string]string{
 	"charts.x-helm.dev/is-default-preset": "true",
 }
 
-func (r *PrometheusReconciler) CreatePreset(cm kmapi.ClusterManager, p *monitoringv1.Prometheus, isDefault bool) error {
+func (r *PrometheusReconciler) CreatePreset(p *monitoringv1.Prometheus, isDefault bool) error {
 	presets := r.GeneratePresetForPrometheus(*p)
 	presetBytes, err := json.Marshal(presets)
 	if err != nil {
 		return err
 	}
 
-	if cm.ManagedByRancher() && !isDefault {
+	if r.d.Federated() && !isDefault {
 		return r.CreateProjectPreset(p, presetBytes)
 	}
 	return r.CreateClusterPreset(presetBytes)
 }
 
-func (r *PrometheusReconciler) CleanupPreset(cm kmapi.ClusterManager, p *monitoringv1.Prometheus, isDefault bool) error {
-	if cm.ManagedByRancher() && !isDefault {
+func (r *PrometheusReconciler) CleanupPreset(p *monitoringv1.Prometheus, isDefault bool) error {
+	if r.d.Federated() && !isDefault {
 		ns, _, err := r.NamespaceForProjectSettings(p)
 		if err != nil {
 			return err

pkg/controllers/servicemonitor/federation_controller.go

@@ -23,6 +23,8 @@ import (
 	"sort"
 	"strings"
 
+	"go.openviz.dev/grafana-tools/pkg/detector"
+
 	"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	core "k8s.io/api/core/v1"
@@ -52,13 +54,15 @@ type FederationReconciler struct {
 	cfg    *rest.Config
 	kc     client.Client
 	scheme *runtime.Scheme
+	d      detector.Detector
 }
 
-func NewFederationReconciler(cfg *rest.Config, kc client.Client) *FederationReconciler {
+func NewFederationReconciler(cfg *rest.Config, kc client.Client, d detector.Detector) *FederationReconciler {
 	return &FederationReconciler{
 		cfg:    cfg,
 		kc:     kc,
 		scheme: kc.Scheme(),
+		d:      d,
 	}
 }
 
@@ -105,7 +109,11 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		return prometheuses[i].Name < prometheuses[j].Name
 	})
 
-	if !clustermeta.IsRancherManaged(r.kc.RESTMapper()) {
+	if ready, err := r.d.Ready(); !ready {
+		return ctrl.Result{}, err
+	}
+
+	if !r.d.Federated() {
 		// non rancher
 		err := r.updateServiceMonitorLabels(prometheuses[0], &svcMon)
 		if err != nil {
@@ -158,7 +166,7 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	var errList []error
 	for _, prom := range promList.Items {
-		isDefault := IsDefaultPrometheus(prom)
+		isDefault := r.d.IsDefault(client.ObjectKeyFromObject(prom))
 
 		if !isDefault && prom.Namespace == req.Namespace {
 			err := fmt.Errorf("federated service monitor can't be in the same namespace with project Prometheus %s/%s", prom.Namespace, prom.Namespace)
@@ -202,15 +210,6 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	return ctrl.Result{}, errors.NewAggregate(errList)
 }
 
-func IsDefaultPrometheus(prom *monitoringv1.Prometheus) bool {
-	expected := client.ObjectKey{
-		Namespace: clustermeta.NamespaceRancherMonitoring,
-		Name:      clustermeta.PrometheusRancherMonitoring,
-	}
-	pk := client.ObjectKeyFromObject(prom)
-	return pk == expected
-}
-
 func (r *FederationReconciler) updateServiceMonitorLabels(prom *monitoringv1.Prometheus, src *monitoringv1.ServiceMonitor) error {
 	vt, err := cu.CreateOrPatch(context.TODO(), r.kc, src, func(in client.Object, createOp bool) client.Object {
 		obj := in.(*monitoringv1.ServiceMonitor)

pkg/detector/api.go

@@ -0,0 +1,169 @@
+/*
+Copyright AppsCode Inc. and Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package detector
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	clustermeta "kmodules.xyz/client-go/cluster"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var GVKPrometheus = schema.GroupVersionKind{
+	Group:   monitoring.GroupName,
+	Version: monitoringv1.Version,
+	Kind:    "Prometheus",
+}
+
+type Detector interface {
+	Ready() (bool, error)
+	RancherManaged_() bool
+	Federated() bool
+	IsDefault(key types.NamespacedName) bool
+}
+
+func New(kc client.Client) Detector {
+	return &lazy{kc: kc}
+}
+
+var errUnknown = errors.New("unknown")
+
+type lazy struct {
+	kc        client.Client
+	delegated Detector
+	mu        sync.Mutex
+}
+
+var _ Detector = &lazy{}
+
+func (l *lazy) detect() error {
+	var list unstructured.UnstructuredList
+	list.SetGroupVersionKind(GVKPrometheus)
+	err := l.kc.List(context.TODO(), &list)
+	if err != nil {
+		return err
+	}
+	if len(list.Items) == 0 {
+		return errUnknown
+	}
+
+	if clustermeta.IsRancherManaged(l.kc.RESTMapper()) {
+		for _, obj := range list.Items {
+			if obj.GetNamespace() == clustermeta.NamespaceRancherMonitoring &&
+				obj.GetName() == clustermeta.PrometheusRancherMonitoring {
+				l.delegated = &federated{} // rancher style federated
+				return nil
+			}
+		}
+
+		// rancher cluster but using prometheus directly
+		l.delegated = &standalone{rancher: true, singleton: len(list.Items) == 1}
+		return nil
+	}
+
+	// using prometheus directly and not rancher managed
+	l.delegated = &standalone{rancher: false, singleton: len(list.Items) == 1}
+	return nil
+}
+
+func (l *lazy) Ready() (bool, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.delegated == nil {
+		err := l.detect()
+		return err == nil, err
+	}
+	return true, nil
+}
+
+func (l *lazy) RancherManaged_() bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.delegated == nil {
+		panic("NotReady")
+	}
+	return l.delegated.RancherManaged_()
+}
+
+func (l *lazy) Federated() bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.delegated == nil {
+		panic("NotReady")
+	}
+	return l.delegated.Federated()
+}
+
+func (l *lazy) IsDefault(key types.NamespacedName) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.delegated == nil {
+		panic("NotReady")
+	}
+	return l.delegated.IsDefault(key)
+}
+
+type federated struct{}
+
+var _ Detector = federated{}
+
+func (f federated) Ready() (bool, error) {
+	return true, nil
+}
+
+func (f federated) RancherManaged_() bool {
+	return true
+}
+
+func (f federated) Federated() bool {
+	return true
+}
+
+func (f federated) IsDefault(key types.NamespacedName) bool {
+	return key.Namespace == clustermeta.NamespaceRancherMonitoring &&
+		key.Name == clustermeta.PrometheusRancherMonitoring
+}
+
+type standalone struct {
+	rancher   bool
+	singleton bool
+}
+
+var _ Detector = standalone{}
+
+func (s standalone) Ready() (bool, error) {
+	return true, nil
+}
+
+func (s standalone) RancherManaged_() bool {
+	return s.rancher
+}
+
+func (s standalone) Federated() bool {
+	return false
+}
+
+func (s standalone) IsDefault(key types.NamespacedName) bool {
+	return s.singleton
+}