Report control plane and worker node stats (#441)

Signed-off-by: Tamal Saha <[email protected]>

apis/identity/v1alpha1/siteinfo_types.go

@@ -76,7 +76,7 @@ type KubernetesInfo struct {
 	Cluster      *kmapi.ClusterMetadata `json:"cluster,omitempty"`
 	Version      *version.Info          `json:"version,omitempty"`
 	ControlPlane *ControlPlaneInfo      `json:"controlPlane,omitempty"`
-	NodeStats    NodeStats              `json:"nodeStats"`
+	NodeStats    NodeInfo               `json:"nodeStats"`
 }
 
 // https://github.com/kmodules/client-go/blob/kubernetes-1.16.3/tools/analytics/analytics.go#L66
@@ -89,6 +89,12 @@ type ControlPlaneInfo struct {
 	NotAfter       metav1.Time `json:"notAfter"`
 }
 
+type NodeInfo struct {
+	NodeStats    `json:",inline"`
+	ControlPlane *NodeStats `json:"controlPlane,omitempty"`
+	Workers      *NodeStats `json:"workers,omitempty"`
+}
+
 type NodeStats struct {
 	Count int `json:"count,omitempty"`
 

crds/identity.k8s.appscode.com_siteinfos.yaml

@@ -125,8 +125,58 @@ spec:
                     description: 'Capacity represents the total resources of a node.
                       More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#capacity'
                     type: object
+                  controlPlane:
+                    properties:
+                      allocatable:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: Allocatable represents the resources of a node
+                          that are available for scheduling. Defaults to Capacity.
+                        type: object
+                      capacity:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Capacity represents the total resources of a
+                          node. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#capacity'
+                        type: object
+                      count:
+                        type: integer
+                    type: object
                   count:
                     type: integer
+                  workers:
+                    properties:
+                      allocatable:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: Allocatable represents the resources of a node
+                          that are available for scheduling. Defaults to Capacity.
+                        type: object
+                      capacity:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Capacity represents the total resources of a
+                          node. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#capacity'
+                        type: object
+                      count:
+                        type: integer
+                    type: object
                 type: object
               version:
                 description: 'Info contains versioning information. TODO: Add []string

go.mod

@@ -29,7 +29,7 @@ require (
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f
 	kmodules.xyz/apiversion v0.2.0
-	kmodules.xyz/client-go v0.30.7
+	kmodules.xyz/client-go v0.30.9
 	kmodules.xyz/crd-schema-fuzz v0.29.1
 	kmodules.xyz/go-containerregistry v0.0.12
 	kmodules.xyz/resource-metrics v0.30.1

go.sum

@@ -340,8 +340,8 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 kmodules.xyz/apiversion v0.2.0 h1:vAQYqZFm4xu4pbB1cAdHbFEPES6EQkcR4wc06xdTOWk=
 kmodules.xyz/apiversion v0.2.0/go.mod h1:oPX8g8LvlPdPX3Yc5YvCzJHQnw3YF/X4/jdW0b1am80=
-kmodules.xyz/client-go v0.30.7 h1:QcZ3pmyK6YZlqtx1gc8IGK1MyrsapaOW14pc6R7klZA=
-kmodules.xyz/client-go v0.30.7/go.mod h1:XL3PDQIXG4s3xNRL2SSxIvi8b2WyMGpn26dFnOBz0j4=
+kmodules.xyz/client-go v0.30.9 h1:wiLivFlqVZOitAqLFEa1n53GkbYYOmiR8upjIHPHrYk=
+kmodules.xyz/client-go v0.30.9/go.mod h1:XL3PDQIXG4s3xNRL2SSxIvi8b2WyMGpn26dFnOBz0j4=
 kmodules.xyz/crd-schema-fuzz v0.29.1 h1:zJTlWYOrT5dsVVHW8HGcnR/vaWfxQfNh11QwTtkYpcs=
 kmodules.xyz/crd-schema-fuzz v0.29.1/go.mod h1:n708z9YQqLMP2KNLQVgBcRJw1QpSWLvpNCEi+KJDOYE=
 kmodules.xyz/go-containerregistry v0.0.12 h1:Tl32QGmSqRVm9PUEb/f3dgDeu9zW5fVzt3qmAFIE37I=

pkg/identity/siteinfo.go

@@ -23,7 +23,7 @@ import (
 
 	clustermeta "kmodules.xyz/client-go/cluster"
 	"kmodules.xyz/client-go/tools/clusterid"
-	auditorapi "kmodules.xyz/resource-metadata/apis/identity/v1alpha1"
+	identityapi "kmodules.xyz/resource-metadata/apis/identity/v1alpha1"
 	"kmodules.xyz/resource-metrics/api"
 
 	"go.bytebuilders.dev/license-verifier/info"
@@ -35,26 +35,26 @@ import (
 	"k8s.io/client-go/rest"
 )
 
-func GetSiteInfo(cfg *rest.Config, kc kubernetes.Interface, nodes []*core.Node, licenseID string) (*auditorapi.SiteInfo, error) {
-	si := auditorapi.SiteInfo{
+func GetSiteInfo(cfg *rest.Config, kc kubernetes.Interface, nodes []*core.Node, licenseID string) (*identityapi.SiteInfo, error) {
+	si := identityapi.SiteInfo{
 		TypeMeta: metav1.TypeMeta{
-			APIVersion: auditorapi.SchemeGroupVersion.String(),
+			APIVersion: identityapi.SchemeGroupVersion.String(),
 			Kind:       "SiteInfo",
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: SelfName,
 		},
-		Kubernetes: &auditorapi.KubernetesInfo{},
+		Kubernetes: &identityapi.KubernetesInfo{},
 	}
 
 	if info.ProductName != "" || v.Version.Version != "" || licenseID != "" {
-		si.Product = &auditorapi.ProductInfo{}
+		si.Product = &identityapi.ProductInfo{}
 		si.Product.LicenseID = licenseID
 		si.Product.ProductOwnerName = info.ProductOwnerName
 		si.Product.ProductOwnerUID = info.ProductOwnerUID
 		si.Product.ProductName = info.ProductName
 		si.Product.ProductUID = info.ProductUID
-		si.Product.Version = auditorapi.Version{
+		si.Product.Version = identityapi.Version{
 			Version:         v.Version.Version,
 			VersionStrategy: v.Version.VersionStrategy,
 			CommitHash:      v.Version.CommitHash,
@@ -81,7 +81,7 @@ func GetSiteInfo(cfg *rest.Config, kc kubernetes.Interface, nodes []*core.Node,
 	if err != nil {
 		return nil, err
 	} else {
-		si.Kubernetes.ControlPlane = &auditorapi.ControlPlaneInfo{
+		si.Kubernetes.ControlPlane = &identityapi.ControlPlaneInfo{
 			NotBefore: metav1.NewTime(cert.NotBefore),
 			NotAfter:  metav1.NewTime(cert.NotAfter),
 			// DNSNames:       cert.DNSNames,
@@ -143,20 +143,41 @@ func GetSiteInfo(cfg *rest.Config, kc kubernetes.Interface, nodes []*core.Node,
 	return &si, nil
 }
 
-func RefreshNodeStats(si *auditorapi.SiteInfo, nodes []*core.Node) {
+func RefreshNodeStats(si *identityapi.SiteInfo, nodes []*core.Node) {
 	if len(nodes) == 0 {
 		return
 	}
-	si.Kubernetes.NodeStats.Count = len(nodes)
 
-	var capacity core.ResourceList
-	var allocatable core.ResourceList
+	var total, cp, w identityapi.NodeStats
 	for _, node := range nodes {
-		capacity = api.AddResourceList(capacity, node.Status.Capacity)
-		allocatable = api.AddResourceList(allocatable, node.Status.Allocatable)
+		total.Count++
+		total.Capacity = api.AddResourceList(total.Capacity, node.Status.Capacity)
+		total.Allocatable = api.AddResourceList(total.Allocatable, node.Status.Allocatable)
+
+		if isControlPlaneNode(node) {
+			cp.Count++
+			cp.Capacity = api.AddResourceList(cp.Capacity, node.Status.Capacity)
+			cp.Allocatable = api.AddResourceList(cp.Allocatable, node.Status.Allocatable)
+		} else {
+			w.Count++
+			w.Capacity = api.AddResourceList(w.Capacity, node.Status.Capacity)
+			w.Allocatable = api.AddResourceList(w.Allocatable, node.Status.Allocatable)
+		}
+	}
+
+	si.Kubernetes.NodeStats.NodeStats = total
+	if cp.Count > 0 {
+		si.Kubernetes.NodeStats.ControlPlane = &cp
 	}
-	si.Kubernetes.NodeStats.Capacity = capacity
-	si.Kubernetes.NodeStats.Allocatable = allocatable
+	if w.Count > 0 {
+		si.Kubernetes.NodeStats.Workers = &w
+	}
+}
+
+func isControlPlaneNode(node *core.Node) bool {
+	_, okCP := node.Labels["node-role.kubernetes.io/control-plane"]
+	_, okMaster := node.Labels["node-role.kubernetes.io/master"]
+	return okCP || okMaster
 }
 
 func skipIP(ip net.IP) bool {

vendor/kmodules.xyz/client-go/apiextensions/controller.go

@@ -21,15 +21,16 @@ import (
 	"sync"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 )
 
 type (
-	SetupFn func(ctx context.Context, mgr ctrl.Manager)
-	TestFn  func(*apiextensionsv1.CustomResourceDefinition) bool
+	SetupFn func(context.Context, ctrl.Manager)
+	TestFn  func(meta.RESTMapper, *apiextensionsv1.CustomResourceDefinition) bool
 )
 
 type setupGroup struct {
@@ -77,7 +78,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if !setupFnExists {
 		return ctrl.Result{}, nil
 	}
-	if !testFns[gk](&crd) {
+	if !testFns[gk](r.mgr.GetRESTMapper(), &crd) {
 		return ctrl.Result{}, nil
 	}
 
@@ -111,7 +112,7 @@ func MultiRegisterSetup(gks []schema.GroupKind, fn SetupFn, tn ...TestFn) {
 	mu.Lock()
 	defer mu.Unlock()
 
-	testFN := andTestFn(tn...)
+	testFN := andTestFn(append(tn, allCRDPresent(gks))...)
 	for _, gk := range gks {
 		setupFns[gk] = setupGroup{
 			gks: gks,
@@ -121,10 +122,26 @@ func MultiRegisterSetup(gks []schema.GroupKind, fn SetupFn, tn ...TestFn) {
 	}
 }
 
+func allCRDPresent(gks []schema.GroupKind) TestFn {
+	return func(mapper meta.RESTMapper, definition *apiextensionsv1.CustomResourceDefinition) bool {
+		for _, gk := range gks {
+			if !crdFound(mapper, gk) {
+				return false
+			}
+		}
+		return true
+	}
+}
+
+func crdFound(mapper meta.RESTMapper, gk schema.GroupKind) bool {
+	_, err := mapper.RESTMappings(gk)
+	return err == nil
+}
+
 func andTestFn(fns ...TestFn) TestFn {
-	return func(crd *apiextensionsv1.CustomResourceDefinition) bool {
+	return func(mapper meta.RESTMapper, crd *apiextensionsv1.CustomResourceDefinition) bool {
 		for _, fn := range fns {
-			if !fn(crd) {
+			if !fn(mapper, crd) {
 				return false
 			}
 		}