Add client org reconciler

Signed-off-by: Tamal Saha <[email protected]>

pkg/apiserver/apiserver.go

@@ -27,6 +27,7 @@ import (
 	"go.openviz.dev/apimachinery/apis/ui"
 	uiinstall "go.openviz.dev/apimachinery/apis/ui/install"
 	uiapi "go.openviz.dev/apimachinery/apis/ui/v1alpha1"
+	namespacecontroller "go.openviz.dev/grafana-tools/pkg/controllers/namespace"
 	promtehsucontroller "go.openviz.dev/grafana-tools/pkg/controllers/prometheus"
 	"go.openviz.dev/grafana-tools/pkg/controllers/ranchertoken"
 	servicemonitorcontroller "go.openviz.dev/grafana-tools/pkg/controllers/servicemonitor"
@@ -202,6 +203,17 @@ func (c completedConfig) New(ctx context.Context) (*UIServer, error) {
 		Group: monitoring.GroupName,
 		Kind:  monitoringv1.PrometheusesKind,
 	}, func(ctx context.Context, mgr ctrl.Manager) {
+		if err = namespacecontroller.NewReconciler(
+			mgr.GetClient(),
+			bc,
+			cid,
+			c.ExtraConfig.HubUID,
+			d,
+		).SetupWithManager(mgr); err != nil {
+			klog.Error(err, "unable to create controller", "controller", "ClientOrg")
+			os.Exit(1)
+		}
+
 		if err = promtehsucontroller.NewReconciler(
 			mgr.GetClient(),
 			bc,

pkg/controllers/namespace/namespace_controller.go

@@ -0,0 +1,368 @@
+package namespace
+
+import (
+	"context"
+	"fmt"
+	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"go.openviz.dev/apimachinery/apis/openviz/v1alpha1"
+	openvizapi "go.openviz.dev/apimachinery/apis/openviz/v1alpha1"
+	"go.openviz.dev/grafana-tools/pkg/controllers/prometheus"
+	"go.openviz.dev/grafana-tools/pkg/detector"
+	core "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
+	kmapi "kmodules.xyz/client-go/api/v1"
+	cu "kmodules.xyz/client-go/client"
+	clustermeta "kmodules.xyz/client-go/cluster"
+	"kmodules.xyz/client-go/meta"
+	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
+	mona "kmodules.xyz/monitoring-agent-api/api/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+// ClientOrgReconciler reconciles a GatewayConfig object
+type ClientOrgReconciler struct {
+	kc         client.Client
+	scheme     *runtime.Scheme
+	bc         *prometheus.Client
+	clusterUID string
+	hubUID     string
+	d          detector.Detector
+}
+
+func NewReconciler(kc client.Client, bc *prometheus.Client, clusterUID, hubUID string, d detector.Detector) *ClientOrgReconciler {
+	return &ClientOrgReconciler{
+		kc:         kc,
+		scheme:     kc.Scheme(),
+		bc:         bc,
+		clusterUID: clusterUID,
+		hubUID:     hubUID,
+		d:          d,
+	}
+}
+
+const (
+	srcRefKey  = "meta.appcode.com/source"
+	srcHashKey = "meta.appcode.com/hash"
+)
+
+func (r *ClientOrgReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := log.FromContext(ctx)
+
+	var ns core.Namespace
+	if err := r.kc.Get(ctx, req.NamespacedName, &ns); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	if ready, err := r.d.Ready(); err != nil || !ready {
+		return ctrl.Result{}, err
+	}
+	if r.d.RancherManaged() && r.d.Federated() {
+		return ctrl.Result{}, fmt.Errorf("client organization mode is not supported when federated prometheus is used in a Rancher managed cluster")
+	}
+
+	// create {client}-monitoring namespace
+	// add finalizer?
+	monNamespace := core.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fmt.Sprintf("%s-monitoring", ns.Name),
+		},
+	}
+	if err := r.kc.Get(ctx, client.ObjectKeyFromObject(&monNamespace), &monNamespace); apierrors.IsNotFound(err) {
+		if err := r.kc.Create(ctx, &monNamespace); err != nil {
+			return ctrl.Result{}, err
+		}
+	} else if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	var dashboardList v1alpha1.GrafanaDashboardList
+	if err := r.kc.List(ctx, &dashboardList); err != nil {
+		return ctrl.Result{}, err
+	}
+	for _, dashboard := range dashboardList.Items {
+		if _, found := dashboard.Annotations[srcRefKey]; found {
+			continue
+		}
+		copiedDashboard := dashboard.DeepCopy()
+		copiedDashboard.Namespace = monNamespace.Name
+
+		opresult, err := controllerutil.CreateOrPatch(ctx, r.kc, copiedDashboard, func() error {
+			if copiedDashboard.Annotations == nil {
+				copiedDashboard.Annotations = map[string]string{}
+			}
+			copiedDashboard.Annotations[srcRefKey] = client.ObjectKeyFromObject(&dashboard).String()
+			copiedDashboard.Annotations[srcHashKey] = meta.ObjectHash(&dashboard)
+
+			// hardcode namespace
+			// datasource
+
+			return nil
+		})
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		if opresult != controllerutil.OperationResultNone {
+			log.Info(fmt.Sprintf("%s GrafanaDashboard %s/%s", opresult, copiedDashboard.Namespace, copiedDashboard.Name))
+		}
+	}
+
+	// confirm trickter sa registered
+	var saList core.ServiceAccountList
+	if err := r.kc.List(ctx, &saList); err != nil {
+		return ctrl.Result{}, err
+	}
+	var saKey types.NamespacedName
+	for _, sa := range saList.Items {
+		if sa.Name != prometheus.ServiceAccountTrickster {
+			continue
+		}
+		if sa.Annotations[prometheus.RegisteredKey] != "" {
+		}
+
+		saKey = types.NamespacedName{
+			Namespace: sa.Namespace,
+			Name:      sa.Name,
+		}
+		break
+	}
+	if saKey.Namespace == "" {
+		return ctrl.Result{}, fmt.Errorf("service account %s is not registered", prometheus.ServiceAccountTrickster)
+	}
+
+	var promList monitoringv1.PrometheusList
+	if err := r.kc.List(ctx, &promList, client.InNamespace(saKey.Namespace)); err != nil {
+		return ctrl.Result{}, err
+	} else if len(promList.Items) == 0 {
+		return ctrl.Result{}, fmt.Errorf("prometheus not found in namespace %s", saKey.Namespace)
+	} else if len(promList.Items) > 1 {
+		return ctrl.Result{}, fmt.Errorf("more than one prometheus found in namespace %s", saKey.Namespace)
+	}
+	// prom := promList.Items[0]
+
+	var promService core.Service
+	err := r.kc.Get(context.TODO(), client.ObjectKeyFromObject(promList.Items[0]), &promService)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	var saToken string
+	var caCrt string
+	s, err := cu.GetServiceAccountTokenSecret(r.kc, saKey)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	saToken = string(s.Data["token"])
+	caCrt = string(s.Data["ca.crt"])
+
+	var pcfg mona.PrometheusConfig
+	pcfg.Service = mona.ServiceSpec{
+		Scheme:    "http",
+		Name:      promService.Name,
+		Namespace: promService.Namespace,
+		Port:      "",
+		Path:      "",
+		Query:     "",
+	}
+	for _, p := range promService.Spec.Ports {
+		if p.Name == prometheus.PortPrometheus {
+			pcfg.Service.Port = fmt.Sprintf("%d", p.Port)
+		}
+	}
+	// pcfg.URL = fmt.Sprintf("%s/api/v1/namespaces/%s/services/%s:%s:%s/proxy/", r.cfg.Host, pcfg.Service.Namespace, pcfg.Service.Scheme, pcfg.Service.Name, pcfg.Service.Port)
+
+	// remove basic auth and client cert auth
+	//if rancherToken != nil {
+	//	pcfg.BearerToken = rancherToken.Token
+	//} else {
+	pcfg.BearerToken = saToken
+	// }
+	pcfg.BasicAuth = mona.BasicAuth{}
+	pcfg.TLS.Cert = ""
+	pcfg.TLS.Key = ""
+	pcfg.TLS.Ca = caCrt
+
+	cm, err := clustermeta.ClusterMetadata(r.kc)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+	state := cm.State()
+
+	applyMarkers := func(in client.Object, createOp bool) client.Object {
+		obj := in.(*core.Namespace)
+		if obj.Annotations == nil {
+			obj.Annotations = map[string]string{}
+		}
+		obj.Annotations[prometheus.RegisteredKey] = state
+		return obj
+	}
+
+	if r.bc != nil &&
+		monNamespace.Annotations[prometheus.RegisteredKey] != state {
+		resp, err := r.bc.Register(mona.PrometheusContext{
+			HubUID:     r.hubUID,
+			ClusterUID: r.clusterUID,
+			ProjectId:  "",
+			Default:    true,
+		}, pcfg)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
+		err = r.CreateGrafanaAppBinding(monNamespace.Name, resp)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
+		rbvt, err := cu.CreateOrPatch(context.TODO(), r.kc, &monNamespace, applyMarkers)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+		klog.Infof("%s namespace %s with %s annotation", rbvt, monNamespace.Name, prometheus.RegisteredKey)
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *ClientOrgReconciler) CreateGrafanaAppBinding(monNamespace string, resp *prometheus.GrafanaDatasourceResponse) error {
+	ab := appcatalog.AppBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "grafana",
+			Namespace: monNamespace,
+		},
+	}
+
+	abvt, err := cu.CreateOrPatch(context.TODO(), r.kc, &ab, func(in client.Object, createOp bool) client.Object {
+		obj := in.(*appcatalog.AppBinding)
+
+		//ref := metav1.NewControllerRef(prom, schema.GroupVersionKind{
+		//	Group:   monitoring.GroupName,
+		//	Version: monitoringv1.Version,
+		//	Kind:    "Prometheus",
+		//})
+		//obj.OwnerReferences = []metav1.OwnerReference{*ref}
+		//
+		//if obj.Annotations == nil {
+		//	obj.Annotations = make(map[string]string)
+		//}
+		//obj.Annotations["monitoring.appscode.com/is-default-grafana"] = "true"
+
+		obj.Spec.Type = "Grafana"
+		obj.Spec.AppRef = nil
+		obj.Spec.ClientConfig = appcatalog.ClientConfig{
+			URL: ptr.To(resp.Grafana.URL),
+			//Service: &appcatalog.ServiceReference{
+			//	Scheme:    "http",
+			//	Namespace: svc.Namespace,
+			//	Name:      svc.Name,
+			//	Port:      0,
+			//	Path:      "",
+			//	Query:     "",
+			//},
+			//InsecureSkipTLSVerify: false,
+			//CABundle:              nil,
+			//ServerName:            "",
+		}
+		obj.Spec.Secret = &core.LocalObjectReference{
+			Name: ab.Name + "-auth",
+		}
+
+		// TODO: handle TLS config returned in resp
+		if caCert := r.bc.CACert(); len(caCert) > 0 {
+			obj.Spec.ClientConfig.CABundle = caCert
+		}
+
+		params := openvizapi.GrafanaConfiguration{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "GrafanaConfiguration",
+				APIVersion: openvizapi.SchemeGroupVersion.String(),
+			},
+			Datasource: resp.Datasource,
+			FolderID:   resp.FolderID,
+		}
+		paramBytes, err := json.Marshal(params)
+		if err != nil {
+			panic(err)
+		}
+		obj.Spec.Parameters = &runtime.RawExtension{
+			Raw: paramBytes,
+		}
+
+		return obj
+	})
+	if err == nil {
+		klog.Infof("%s AppBinding %s/%s", abvt, ab.Namespace, ab.Name)
+
+		authSecret := core.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      ab.Name + "-auth",
+				Namespace: monNamespace,
+			},
+		}
+
+		svt, e2 := cu.CreateOrPatch(context.TODO(), r.kc, &authSecret, func(in client.Object, createOp bool) client.Object {
+			obj := in.(*core.Secret)
+
+			ref := metav1.NewControllerRef(&ab, schema.GroupVersionKind{
+				Group:   appcatalog.SchemeGroupVersion.Group,
+				Version: appcatalog.SchemeGroupVersion.Version,
+				Kind:    "AppBinding",
+			})
+			obj.OwnerReferences = []metav1.OwnerReference{*ref}
+
+			obj.StringData = map[string]string{
+				"token": resp.Grafana.BearerToken,
+			}
+
+			return obj
+		})
+		if e2 == nil {
+			klog.Infof("%s Grafana auth secret %s/%s", svt, authSecret.Namespace, authSecret.Name)
+		}
+	}
+
+	return err
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *ClientOrgReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&core.Namespace{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
+			return obj.GetLabels()[kmapi.ClientOrgKey] == "true"
+		}))).
+		Watches(&core.ServiceAccount{}, handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, _ client.Object) []reconcile.Request {
+			var list core.NamespaceList
+			if err := r.kc.List(ctx, &list, client.MatchingLabels{
+				kmapi.ClientOrgKey: "true",
+			}); err != nil {
+				return nil
+			}
+			reqs := make([]reconcile.Request, 0, len(list.Items))
+			for _, ns := range list.Items {
+				reqs = append(reqs, reconcile.Request{
+					NamespacedName: types.NamespacedName{
+						Name: ns.Name,
+					},
+				})
+			}
+			return reqs
+		}), builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
+			return obj.GetName() == prometheus.ServiceAccountTrickster &&
+				obj.GetLabels()[prometheus.RegisteredKey] != ""
+		}))).
+		Named("namespace").
+		Complete(r)
+}