Skip to content

Commit

Permalink
Handle rancher cluster with kube-prometheus-stack deployment (#133)
Browse files Browse the repository at this point in the history
Signed-off-by: Tamal Saha <[email protected]>
  • Loading branch information
tamalsaha authored Apr 17, 2024
1 parent 3d72097 commit 98138bc
Show file tree
Hide file tree
Showing 4 changed files with 201 additions and 39 deletions.
5 changes: 5 additions & 0 deletions pkg/apiserver/apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
uiapi "go.openviz.dev/apimachinery/apis/ui/v1alpha1"
promtehsucontroller "go.openviz.dev/grafana-tools/pkg/controllers/prometheus"
servicemonitorcontroller "go.openviz.dev/grafana-tools/pkg/controllers/servicemonitor"
"go.openviz.dev/grafana-tools/pkg/detector"
dashgroupstorage "go.openviz.dev/grafana-tools/pkg/registry/ui/dashboardgroup"

"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
Expand Down Expand Up @@ -183,6 +184,8 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
os.Exit(1)
}

d := detector.New(mgr.GetClient())

apiextensions.RegisterSetup(schema.GroupKind{
Group: monitoring.GroupName,
Kind: monitoringv1.PrometheusesKind,
Expand All @@ -191,6 +194,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
mgr.GetClient(),
bc,
cid,
d,
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", "controller", "Prometheus")
os.Exit(1)
Expand All @@ -212,6 +216,7 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
if err = servicemonitorcontroller.NewFederationReconciler(
c.ExtraConfig.ClientConfig,
mgr.GetClient(),
d,
).SetupWithManager(mgr); err != nil {
klog.Error(err, "unable to create controller", " federation controller", "ServiceMonitor")
os.Exit(1)
Expand Down
43 changes: 16 additions & 27 deletions pkg/controllers/prometheus/prometheus_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"fmt"

openvizapi "go.openviz.dev/apimachinery/apis/openviz/v1alpha1"
"go.openviz.dev/grafana-tools/pkg/detector"

"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
Expand Down Expand Up @@ -53,14 +54,16 @@ type PrometheusReconciler struct {
scheme *runtime.Scheme
bc *Client
clusterUID string
d detector.Detector
}

func NewReconciler(kc client.Client, bc *Client, clusterUID string) *PrometheusReconciler {
func NewReconciler(kc client.Client, bc *Client, clusterUID string, d detector.Detector) *PrometheusReconciler {
return &PrometheusReconciler{
kc: kc,
scheme: kc.Scheme(),
bc: bc,
clusterUID: clusterUID,
d: d,
}
}

Expand All @@ -85,21 +88,15 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, client.IgnoreNotFound(err)
}

cm := clustermeta.DetectClusterManager(r.kc)
gvk := schema.GroupVersionKind{
Group: monitoring.GroupName,
Version: monitoringv1.Version,
Kind: "Prometheus",
}

key := client.ObjectKeyFromObject(&prom)
isDefault, err := r.IsDefault(cm, gvk, key)
if err != nil {
if ready, err := r.d.Ready(); !ready {
return ctrl.Result{}, err
}

key := req.NamespacedName
isDefault := r.d.IsDefault(key)

if prom.DeletionTimestamp != nil {
err := r.CleanupPreset(cm, &prom, isDefault)
err := r.CleanupPreset(&prom, isDefault)
if err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -146,22 +143,14 @@ func (r *PrometheusReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
klog.Infof("%s Prometheus %s/%s to add finalizer %s", vt, prom.Namespace, prom.Name, mona.PrometheusKey)

if err := r.SetupClusterForPrometheus(cm, &prom, isDefault); err != nil {
if err := r.SetupClusterForPrometheus(&prom, isDefault); err != nil {
log.Error(err, "unable to setup Prometheus")
return ctrl.Result{}, err
}

return ctrl.Result{}, nil
}

func (r *PrometheusReconciler) IsDefault(cm kmapi.ClusterManager, gvk schema.GroupVersionKind, key types.NamespacedName) (bool, error) {
if cm.ManagedByRancher() {
return key.Namespace == clustermeta.NamespaceRancherMonitoring &&
key.Name == clustermeta.PrometheusRancherMonitoring, nil
}
return clustermeta.IsSingletonResource(r.kc, gvk, key)
}

func (r *PrometheusReconciler) findServiceForPrometheus(key types.NamespacedName) (*core.Service, error) {
var svc core.Service
err := r.kc.Get(context.TODO(), key, &svc)
Expand All @@ -176,7 +165,7 @@ const (
saTrickster = "trickster"
)

func (r *PrometheusReconciler) SetupClusterForPrometheus(cm kmapi.ClusterManager, prom *monitoringv1.Prometheus, isDefault bool) error {
func (r *PrometheusReconciler) SetupClusterForPrometheus(prom *monitoringv1.Prometheus, isDefault bool) error {
key := client.ObjectKeyFromObject(prom)

svc, err := r.findServiceForPrometheus(key)
Expand Down Expand Up @@ -273,7 +262,7 @@ func (r *PrometheusReconciler) SetupClusterForPrometheus(cm kmapi.ClusterManager
}
klog.Infof("%s role binding %s/%s", rbvt, rb.Namespace, rb.Name)

err = r.CreatePreset(cm, prom, isDefault)
err = r.CreatePreset(prom, isDefault)
if err != nil {
return err
}
Expand Down Expand Up @@ -362,21 +351,21 @@ var defaultPresetsLabels = map[string]string{
"charts.x-helm.dev/is-default-preset": "true",
}

func (r *PrometheusReconciler) CreatePreset(cm kmapi.ClusterManager, p *monitoringv1.Prometheus, isDefault bool) error {
func (r *PrometheusReconciler) CreatePreset(p *monitoringv1.Prometheus, isDefault bool) error {
presets := r.GeneratePresetForPrometheus(*p)
presetBytes, err := json.Marshal(presets)
if err != nil {
return err
}

if cm.ManagedByRancher() && !isDefault {
if r.d.Federated() && !isDefault {
return r.CreateProjectPreset(p, presetBytes)
}
return r.CreateClusterPreset(presetBytes)
}

func (r *PrometheusReconciler) CleanupPreset(cm kmapi.ClusterManager, p *monitoringv1.Prometheus, isDefault bool) error {
if cm.ManagedByRancher() && !isDefault {
func (r *PrometheusReconciler) CleanupPreset(p *monitoringv1.Prometheus, isDefault bool) error {
if r.d.Federated() && !isDefault {
ns, _, err := r.NamespaceForProjectSettings(p)
if err != nil {
return err
Expand Down
23 changes: 11 additions & 12 deletions pkg/controllers/servicemonitor/federation_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"sort"
"strings"

"go.openviz.dev/grafana-tools/pkg/detector"

"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
core "k8s.io/api/core/v1"
Expand Down Expand Up @@ -52,13 +54,15 @@ type FederationReconciler struct {
cfg *rest.Config
kc client.Client
scheme *runtime.Scheme
d detector.Detector
}

func NewFederationReconciler(cfg *rest.Config, kc client.Client) *FederationReconciler {
func NewFederationReconciler(cfg *rest.Config, kc client.Client, d detector.Detector) *FederationReconciler {
return &FederationReconciler{
cfg: cfg,
kc: kc,
scheme: kc.Scheme(),
d: d,
}
}

Expand Down Expand Up @@ -105,7 +109,11 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return prometheuses[i].Name < prometheuses[j].Name
})

if !clustermeta.IsRancherManaged(r.kc.RESTMapper()) {
if ready, err := r.d.Ready(); !ready {
return ctrl.Result{}, err
}

if !r.d.Federated() {
// non rancher
err := r.updateServiceMonitorLabels(prometheuses[0], &svcMon)
if err != nil {
Expand Down Expand Up @@ -158,7 +166,7 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request)

var errList []error
for _, prom := range promList.Items {
isDefault := IsDefaultPrometheus(prom)
isDefault := r.d.IsDefault(client.ObjectKeyFromObject(prom))

if !isDefault && prom.Namespace == req.Namespace {
err := fmt.Errorf("federated service monitor can't be in the same namespace with project Prometheus %s/%s", prom.Namespace, prom.Namespace)
Expand Down Expand Up @@ -202,15 +210,6 @@ func (r *FederationReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return ctrl.Result{}, errors.NewAggregate(errList)
}

func IsDefaultPrometheus(prom *monitoringv1.Prometheus) bool {
expected := client.ObjectKey{
Namespace: clustermeta.NamespaceRancherMonitoring,
Name: clustermeta.PrometheusRancherMonitoring,
}
pk := client.ObjectKeyFromObject(prom)
return pk == expected
}

func (r *FederationReconciler) updateServiceMonitorLabels(prom *monitoringv1.Prometheus, src *monitoringv1.ServiceMonitor) error {
vt, err := cu.CreateOrPatch(context.TODO(), r.kc, src, func(in client.Object, createOp bool) client.Object {
obj := in.(*monitoringv1.ServiceMonitor)
Expand Down
169 changes: 169 additions & 0 deletions pkg/detector/api.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/*
Copyright AppsCode Inc. and Contributors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package detector

import (
"context"
"errors"
"sync"

"github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring"
monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
clustermeta "kmodules.xyz/client-go/cluster"
"sigs.k8s.io/controller-runtime/pkg/client"
)

var GVKPrometheus = schema.GroupVersionKind{
Group: monitoring.GroupName,
Version: monitoringv1.Version,
Kind: "Prometheus",
}

type Detector interface {
Ready() (bool, error)
RancherManaged_() bool
Federated() bool
IsDefault(key types.NamespacedName) bool
}

func New(kc client.Client) Detector {
return &lazy{kc: kc}
}

var errUnknown = errors.New("unknown")

type lazy struct {
kc client.Client
delegated Detector
mu sync.Mutex
}

var _ Detector = &lazy{}

func (l *lazy) detect() error {
var list unstructured.UnstructuredList
list.SetGroupVersionKind(GVKPrometheus)
err := l.kc.List(context.TODO(), &list)
if err != nil {
return err
}
if len(list.Items) == 0 {
return errUnknown
}

if clustermeta.IsRancherManaged(l.kc.RESTMapper()) {
for _, obj := range list.Items {
if obj.GetNamespace() == clustermeta.NamespaceRancherMonitoring &&
obj.GetName() == clustermeta.PrometheusRancherMonitoring {
l.delegated = &federated{} // rancher style federated
return nil
}
}

// rancher cluster but using prometheus directly
l.delegated = &standalone{rancher: true, singleton: len(list.Items) == 1}
return nil
}

// using prometheus directly and not rancher managed
l.delegated = &standalone{rancher: false, singleton: len(list.Items) == 1}
return nil
}

func (l *lazy) Ready() (bool, error) {
l.mu.Lock()
defer l.mu.Unlock()
if l.delegated == nil {
err := l.detect()
return err == nil, err
}
return true, nil
}

func (l *lazy) RancherManaged_() bool {
l.mu.Lock()
defer l.mu.Unlock()
if l.delegated == nil {
panic("NotReady")
}
return l.delegated.RancherManaged_()
}

func (l *lazy) Federated() bool {
l.mu.Lock()
defer l.mu.Unlock()
if l.delegated == nil {
panic("NotReady")
}
return l.delegated.Federated()
}

func (l *lazy) IsDefault(key types.NamespacedName) bool {
l.mu.Lock()
defer l.mu.Unlock()
if l.delegated == nil {
panic("NotReady")
}
return l.delegated.IsDefault(key)
}

type federated struct{}

var _ Detector = federated{}

func (f federated) Ready() (bool, error) {
return true, nil
}

func (f federated) RancherManaged_() bool {
return true
}

func (f federated) Federated() bool {
return true
}

func (f federated) IsDefault(key types.NamespacedName) bool {
return key.Namespace == clustermeta.NamespaceRancherMonitoring &&
key.Name == clustermeta.PrometheusRancherMonitoring
}

type standalone struct {
rancher bool
singleton bool
}

var _ Detector = standalone{}

func (s standalone) Ready() (bool, error) {
return true, nil
}

func (s standalone) RancherManaged_() bool {
return s.rancher
}

func (s standalone) Federated() bool {
return false
}

func (s standalone) IsDefault(key types.NamespacedName) bool {
return s.singleton
}

0 comments on commit 98138bc

Please sign in to comment.