Reconcile Grafana when a cluster is re-imported

Signed-off-by: Tamal Saha <[email protected]>

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/zeebo/xxh3 v1.0.2
 	go.bytebuilders.dev/license-verifier v0.14.1
-	go.openviz.dev/apimachinery v0.0.7
+	go.openviz.dev/apimachinery v0.0.8
 	go.openviz.dev/grafana-sdk v0.0.5
 	golang.org/x/text v0.16.0
 	gomodules.xyz/logs v0.0.7
@@ -30,8 +30,9 @@ require (
 	k8s.io/apiserver v0.30.2
 	k8s.io/client-go v0.30.2
 	k8s.io/klog/v2 v2.130.1
+	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	kmodules.xyz/authorizer v0.29.1
-	kmodules.xyz/client-go v0.30.10
+	kmodules.xyz/client-go v0.30.11
 	kmodules.xyz/custom-resources v0.30.0
 	kmodules.xyz/monitoring-agent-api v0.30.0
 	sigs.k8s.io/controller-runtime v0.18.4
@@ -45,7 +46,7 @@ require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/PuerkitoBio/purell v1.2.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
-	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
@@ -70,7 +71,7 @@ require (
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/btree v1.0.1 // indirect
+	github.com/google/btree v1.1.2 // indirect
 	github.com/google/cel-go v0.17.8 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
@@ -154,7 +155,6 @@ require (
 	k8s.io/component-helpers v0.29.0 // indirect
 	k8s.io/kms v0.30.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect
-	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
 	kmodules.xyz/apiversion v0.2.0 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.sum

@@ -35,8 +35,8 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df/g
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0S6Vi7/lbWECcX0j45yZReDZ56BQsrVBOEEY=
-github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
+github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ=
+github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -172,8 +172,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
+github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/cel-go v0.17.8 h1:j9m730pMZt1Fc4oKhCLUHfjj6527LuhYcYw0Rl8gqto=
 github.com/google/cel-go v0.17.8/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
@@ -250,8 +250,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
-github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
+github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -466,8 +466,8 @@ go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om
 go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU=
 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
 go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
-go.openviz.dev/apimachinery v0.0.7 h1:LBW/RtI21FA9ILHEfOMCF+YCSBw2UEHd8H+cgmtZaAs=
-go.openviz.dev/apimachinery v0.0.7/go.mod h1:OUaaMC3iCOyp7R6j5puYMV7fkSNu6ERW2P+jcJ+n1iE=
+go.openviz.dev/apimachinery v0.0.8 h1:lGFIgn+dU+bf7clBBXcPM2/wgllhqEspDUYdoHw9Hxg=
+go.openviz.dev/apimachinery v0.0.8/go.mod h1:I+wL5YJR4U1qQxHNy5yohuB4k43u8W8GbZET7q6wFj0=
 go.openviz.dev/grafana-sdk v0.0.5 h1:87qihU9GvzaJN/uuhWnBzlUsQs+OtjLgfhEr1WbKvPY=
 go.openviz.dev/grafana-sdk v0.0.5/go.mod h1:yP3DAgyHO1Y1VcvuotEvYmHpDWyFHSOI5dEfVuKh9aE=
 go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY=
@@ -762,8 +762,8 @@ kmodules.xyz/apiversion v0.2.0 h1:vAQYqZFm4xu4pbB1cAdHbFEPES6EQkcR4wc06xdTOWk=
 kmodules.xyz/apiversion v0.2.0/go.mod h1:oPX8g8LvlPdPX3Yc5YvCzJHQnw3YF/X4/jdW0b1am80=
 kmodules.xyz/authorizer v0.29.1 h1:uByGGoryKbZcfiEAhjcK/Y345I9mygNQP7DVpkMbNQQ=
 kmodules.xyz/authorizer v0.29.1/go.mod h1:kZRhclL8twzyt2bQuJQJbpYww2sc+qFr8I5PPoq/sWY=
-kmodules.xyz/client-go v0.30.10 h1:adcwugZgT6NNeawHxy2XDbkcd+KXVQD4d3ROaOu0TPU=
-kmodules.xyz/client-go v0.30.10/go.mod h1:XL3PDQIXG4s3xNRL2SSxIvi8b2WyMGpn26dFnOBz0j4=
+kmodules.xyz/client-go v0.30.11 h1:2UFIu18O5xwKyKbmAn+vs8CIcnfogu0IACurgR2HAmU=
+kmodules.xyz/client-go v0.30.11/go.mod h1:XL3PDQIXG4s3xNRL2SSxIvi8b2WyMGpn26dFnOBz0j4=
 kmodules.xyz/crd-schema-fuzz v0.29.1 h1:zJTlWYOrT5dsVVHW8HGcnR/vaWfxQfNh11QwTtkYpcs=
 kmodules.xyz/crd-schema-fuzz v0.29.1/go.mod h1:n708z9YQqLMP2KNLQVgBcRJw1QpSWLvpNCEi+KJDOYE=
 kmodules.xyz/custom-resources v0.30.0 h1:vR3CbseHMLwR4GvtcJJuRuwIV8voKqFqNii27rMcm1o=

pkg/controllers/openviz/grafanadashboard_controller.go

@@ -18,9 +18,12 @@ package openviz
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
 	openvizapi "go.openviz.dev/apimachinery/apis/openviz/v1alpha1"
@@ -31,13 +34,16 @@ import (
 	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/klog/v2"
+	"k8s.io/utils/ptr"
 	kmapi "kmodules.xyz/client-go/api/v1"
 	kmc "kmodules.xyz/client-go/client"
 	condutil "kmodules.xyz/client-go/conditions"
 	meta_util "kmodules.xyz/client-go/meta"
 	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
+	mona "kmodules.xyz/monitoring-agent-api/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -50,6 +56,7 @@ import (
 
 const (
 	GrafanaDashboardFinalizer = "grafanadashboard.openviz.dev/finalizer"
+	registeredKey             = mona.GroupName + "/registered"
 )
 
 // GrafanaDashboardReconciler reconciles a GrafanaDashboard object
@@ -156,7 +163,7 @@ func (r *GrafanaDashboardReconciler) handleSetDashboardError(ctx context.Context
 func (r *GrafanaDashboardReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	appHandler := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request {
 		var dashboardList openvizapi.GrafanaDashboardList
-		err := r.Client.List(ctx, &dashboardList, client.InNamespace(a.GetNamespace()))
+		err := r.Client.List(ctx, &dashboardList)
 		if err != nil {
 			return nil
 		}
@@ -167,22 +174,94 @@ func (r *GrafanaDashboardReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			if err != nil {
 				return nil
 			}
+
 			if ab.Name == a.GetName() &&
 				ab.Namespace == a.GetNamespace() &&
-				db.Status.Phase == openvizapi.GrafanaPhaseFailed {
+				(db.Status.Phase == openvizapi.GrafanaPhaseFailed ||
+					MustIsStateChanged(&db, a.(*appcatalog.AppBinding))) {
 				req = append(req, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&db)})
 			}
 		}
 		return req
 	})
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&openvizapi.GrafanaDashboard{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
-			return !meta_util.MustAlreadyReconciled(obj)
+			db := obj.(*openvizapi.GrafanaDashboard)
+			if !meta_util.MustAlreadyReconciled(obj) {
+				return true
+			}
+			if ab, err := openvizapi.GetGrafana(context.TODO(), r.Client, db.Spec.GrafanaRef.WithNamespace(db.Namespace)); err == nil && MustIsStateChanged(db, ab) {
+				return true
+			}
+			return false
 		}))).
 		Watches(&appcatalog.AppBinding{}, appHandler).
 		Complete(r)
 }
 
+func MustIsStateChanged(db *openvizapi.GrafanaDashboard, ab *appcatalog.AppBinding) bool {
+	changed, _ := IsStateChanged(db, ab)
+	return changed
+}
+
+func IsStateChanged(db *openvizapi.GrafanaDashboard, ab *appcatalog.AppBinding) (bool, error) {
+	if db.Status.Dashboard == nil || db.Status.Dashboard.State == nil {
+		return true, nil
+	}
+	curState, err := GrafanaConnectionHash(ab)
+	if err != nil {
+		return false, err
+	}
+	return curState != *db.Status.Dashboard.State, nil
+}
+
+type stateKey struct {
+	uid        types.UID
+	generation int64
+}
+type stateValue struct {
+	state string
+	err   error
+}
+
+var (
+	stateMap = map[stateKey]stateValue{}
+	stateMu  sync.Mutex
+)
+
+func GrafanaConnectionHash(ab *appcatalog.AppBinding) (string, error) {
+	stateMu.Lock()
+	defer stateMu.Unlock()
+
+	key := stateKey{uid: ab.UID, generation: ab.Generation}
+	result, ok := stateMap[key]
+	if ok {
+		return result.state, result.err
+	}
+
+	result.state, result.err = func() (string, error) {
+		dsConfig := &openvizapi.GrafanaConfiguration{}
+		if ab.Spec.Parameters != nil {
+			if err := json.Unmarshal(ab.Spec.Parameters.Raw, dsConfig); err != nil {
+				return "", err
+			}
+		}
+
+		u, err := ab.URL()
+		if err != nil {
+			return "", err
+		}
+
+		state := fmt.Sprintf("%s,%s,%d", u, dsConfig.Datasource, ptr.Deref(dsConfig.FolderID, 0))
+		h := sha256.New()
+		h.Write([]byte(state))
+		return base64.URLEncoding.EncodeToString(h.Sum(nil)), nil
+	}()
+	stateMap[key] = result
+	return result.state, result.err
+}
+
 func (r *GrafanaDashboardReconciler) deleteExternalDashboard(ctx context.Context, obj *openvizapi.GrafanaDashboard) error {
 	if obj.Status.Dashboard != nil && obj.Status.Dashboard.UID != nil {
 		gc, err := grafana.NewGrafanaClient(ctx, r.Client, obj.Spec.GrafanaRef.WithNamespace(obj.Namespace))
@@ -257,16 +336,22 @@ func (r *GrafanaDashboardReconciler) setDashboard(ctx context.Context, obj *open
 		return r.handleSetDashboardError(ctx, obj, err, false)
 	}
 
+	state, err := GrafanaConnectionHash(ab)
+	if err != nil {
+		return r.handleSetDashboardError(ctx, obj, err, false)
+	}
+
 	_, err = kmc.PatchStatus(ctx, r.Client, obj, func(obj client.Object) client.Object {
 		in := obj.(*openvizapi.GrafanaDashboard)
 		reason := "Dashboard is successfully created"
 		in.Status.Dashboard = &openvizapi.GrafanaDashboardReference{
-			ID:      pointer.Int64P(int64(pointer.Int(resp.ID))),
+			ID:      ptr.To(int64(*resp.ID)),
 			UID:     resp.UID,
 			Slug:    resp.Slug,
 			URL:     resp.URL,
-			OrgID:   pointer.Int64P(int64(pointer.Int(orgId.ID))),
-			Version: pointer.Int64P(int64(pointer.Int(resp.Version))),
+			OrgID:   ptr.To(int64(*orgId.ID)),
+			Version: ptr.To(int64(*resp.Version)),
+			State:   ptr.To(state),
 		}
 		in.Status.Phase = openvizapi.GrafanaPhaseCurrent
 		in.Status.ObservedGeneration = in.Generation

pkg/controllers/prometheus/prometheus_controller.go

@@ -44,7 +44,9 @@ import (
 	mona "kmodules.xyz/monitoring-agent-api/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	chartsapi "x-helm.dev/apimachinery/apis/charts/v1alpha1"
 )
 
@@ -297,7 +299,28 @@ func (r *PrometheusReconciler) SetupClusterForPrometheus(prom *monitoringv1.Prom
 	pcfg.BearerToken = string(s.Data["token"])
 	pcfg.TLS.Ca = string(s.Data["ca.crt"])
 
-	if r.bc != nil && sa.Annotations[registeredKey] != "true" {
+	cm, err := clustermeta.ClusterMetadata(r.kc)
+	if err != nil {
+		return err
+	}
+	state := cm.State()
+
+	// fix legacy deployments
+	if sa.Annotations[registeredKey] == "true" {
+		savt, err = cu.CreateOrPatch(context.TODO(), r.kc, &sa, func(in client.Object, createOp bool) client.Object {
+			obj := in.(*core.ServiceAccount)
+			obj.Annotations = meta_util.OverwriteKeys(obj.Annotations, map[string]string{
+				registeredKey: state,
+			})
+			return obj
+		})
+		if err != nil {
+			return err
+		}
+		klog.Infof("%s service account %s/%s with %s annotation", savt, sa.Namespace, sa.Name, registeredKey)
+
+		return nil
+	} else if r.bc != nil && sa.Annotations[registeredKey] != state {
 		var projectId string
 		if !isDefault {
 			_, projectId, err = r.NamespaceForProjectSettings(prom)
@@ -330,7 +353,7 @@ func (r *PrometheusReconciler) SetupClusterForPrometheus(prom *monitoringv1.Prom
 		savt, err = cu.CreateOrPatch(context.TODO(), r.kc, &sa, func(in client.Object, createOp bool) client.Object {
 			obj := in.(*core.ServiceAccount)
 			obj.Annotations = meta_util.OverwriteKeys(obj.Annotations, map[string]string{
-				registeredKey: "true",
+				registeredKey: state,
 			})
 			return obj
 		})
@@ -669,7 +692,26 @@ func (r *PrometheusReconciler) CreateGrafanaAppBinding(prom *monitoringv1.Promet
 
 // SetupWithManager sets up the controller with the Manager.
 func (r *PrometheusReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	stateHandler := handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, a client.Object) []reconcile.Request {
+		if a.GetNamespace() != metav1.NamespacePublic || a.GetName() != kmapi.AceInfoConfigMapName {
+			return nil
+		}
+
+		var promList monitoringv1.PrometheusList
+		err := r.kc.List(ctx, &promList)
+		if err != nil {
+			return nil
+		}
+
+		var req []reconcile.Request
+		for _, prom := range promList.Items {
+			req = append(req, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(prom)})
+		}
+		return req
+	})
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&monitoringv1.Prometheus{}).
+		Watches(&core.ConfigMap{}, stateHandler).
 		Complete(r)
 }