From d479370c1bc57531b1973cd57a5e6843b9809c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20R=C3=BCger?= Date: Tue, 6 Aug 2024 17:33:59 +0200 Subject: [PATCH] feat: Use PartialObjectMetadata for Configmaps and Secrets --- internal/store/builder.go | 70 ++++++++++++++++++++++++++++++--- internal/store/configmap.go | 24 +++++------ pkg/app/server.go | 7 ++++ pkg/builder/builder.go | 11 ++++++ pkg/builder/types/interfaces.go | 11 ++++++ pkg/util/utils.go | 30 ++++++++++++++ 6 files changed, 136 insertions(+), 17 deletions(-) diff --git a/internal/store/builder.go b/internal/store/builder.go index db4e06baa9..ff8deb78a1 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -38,7 +38,9 @@ import ( policyv1 "k8s.io/api/policy/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" @@ -65,9 +67,10 @@ var _ ksmtypes.BuilderInterface = &Builder{} // Builder helps to build store. It follows the builder pattern // (https://en.wikipedia.org/wiki/Builder_pattern). type Builder struct { - kubeClient clientset.Interface - customResourceClients map[string]interface{} - namespaces options.NamespaceList + kubeClient clientset.Interface + metadataOnlyKubeClient metadata.Interface + customResourceClients map[string]interface{} + namespaces options.NamespaceList // namespaceFilter is inside fieldSelectorFilter fieldSelectorFilter string ctx context.Context @@ -78,6 +81,7 @@ type Builder struct { shard int32 totalShards int buildStoresFunc ksmtypes.BuildStoresFunc + buildMetadataOnlyStoresFunc ksmtypes.BuildMetadataOnlyStoresFunc buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc allowAnnotationsList map[string][]string allowLabelsList map[string][]string @@ -157,6 +161,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) { b.kubeClient = c } +// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder. +func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) { + b.metadataOnlyKubeClient = c +} + // WithCustomResourceClients sets the customResourceClients property of a Builder. func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) { b.customResourceClients = cs @@ -178,6 +187,11 @@ func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) { b.buildStoresFunc = f } +// WithGenerateMetadataOnlyStoresFunc configures a custom generate custom resource store function +func (b *Builder) WithGenerateMetadataOnlyStoresFunc(f ksmtypes.BuildMetadataOnlyStoresFunc) { + b.buildMetadataOnlyStoresFunc = f +} + // WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) { b.buildCustomResourceStoresFunc = f @@ -188,6 +202,11 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc { return b.buildStores } +// DefaultGenerateMetadataOnlyStoresFunc returns default buildStores function +func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc { + return b.buildMetadataOnlyStores +} + // DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc { return b.buildCustomResourceStores @@ -362,7 +381,7 @@ func availableResources() []string { } func (b *Builder) buildConfigMapStores() []cache.Store { - return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache) + return b.buildMetadataOnlyStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &metav1.PartialObjectMetadata{}, createConfigMapListWatch, b.useAPIServerCache) } func (b *Builder) buildCronJobStores() []cache.Store { @@ -519,7 +538,8 @@ func (b *Builder) buildStores( if b.fieldSelectorFilter != "" { klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) } - listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter) + kubeClient := b.kubeClient + listWatcher := listWatchFunc(kubeClient, v1.NamespaceAll, b.fieldSelectorFilter) b.startReflector(expectedType, store, listWatcher, useAPIServerCache) return []cache.Store{store} } @@ -541,6 +561,46 @@ func (b *Builder) buildStores( return stores } +func (b *Builder) buildMetadataOnlyStores( + metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store { + metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies) + familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies) + + if b.namespaces.IsAllNamespaces() { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + if b.fieldSelectorFilter != "" { + klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) + } + listWatcher := listWatchFunc(b.metadataOnlyKubeClient, v1.NamespaceAll, b.fieldSelectorFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + return []cache.Store{store} + } + + stores := make([]cache.Store, 0, len(b.namespaces)) + for _, ns := range b.namespaces { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + if b.fieldSelectorFilter != "" { + klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter) + } + listWatcher := listWatchFunc(b.metadataOnlyKubeClient, ns, b.fieldSelectorFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } + + return stores +} + // TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores` func (b *Builder) buildCustomResourceStores(resourceName string, metricFamilies []generator.FamilyGenerator, diff --git a/internal/store/configmap.go b/internal/store/configmap.go index dff11ea513..1026f5f165 100644 --- a/internal/store/configmap.go +++ b/internal/store/configmap.go @@ -19,11 +19,11 @@ package store import ( "context" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/watch" - clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" "k8s.io/client-go/tools/cache" basemetrics "k8s.io/component-base/metrics" @@ -43,7 +43,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g metric.Gauge, basemetrics.ALPHA, "", - wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family { if len(allowAnnotationsList) == 0 { return &metric.Family{} } @@ -65,7 +65,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g metric.Gauge, basemetrics.STABLE, "", - wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family { if len(allowLabelsList) == 0 { return &metric.Family{} } @@ -87,7 +87,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g metric.Gauge, basemetrics.STABLE, "", - wrapConfigMapFunc(func(_ *v1.ConfigMap) *metric.Family { + wrapConfigMapFunc(func(_ *metav1.PartialObjectMetadata) *metric.Family { return &metric.Family{ Metrics: []*metric.Metric{{ LabelKeys: []string{}, @@ -103,7 +103,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g metric.Gauge, basemetrics.STABLE, "", - wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family { ms := []*metric.Metric{} if !c.CreationTimestamp.IsZero() { @@ -125,7 +125,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g metric.Gauge, basemetrics.ALPHA, "", - wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family { + wrapConfigMapFunc(func(c *metav1.PartialObjectMetadata) *metric.Family { return &metric.Family{ Metrics: resourceVersionMetric(c.ObjectMeta.ResourceVersion), } @@ -134,22 +134,22 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g } } -func createConfigMapListWatch(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher { +func createConfigMapListWatch(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher { return &cache.ListWatch{ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) { opts.FieldSelector = fieldSelector - return kubeClient.CoreV1().ConfigMaps(ns).List(context.TODO(), opts) + return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).List(context.TODO(), opts) }, WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) { opts.FieldSelector = fieldSelector - return kubeClient.CoreV1().ConfigMaps(ns).Watch(context.TODO(), opts) + return kubeClient.Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}).Namespace(ns).Watch(context.TODO(), opts) }, } } -func wrapConfigMapFunc(f func(*v1.ConfigMap) *metric.Family) func(interface{}) *metric.Family { +func wrapConfigMapFunc(f func(*metav1.PartialObjectMetadata) *metric.Family) func(interface{}) *metric.Family { return func(obj interface{}) *metric.Family { - configMap := obj.(*v1.ConfigMap) + configMap := obj.(*metav1.PartialObjectMetadata) metricFamily := f(configMap) diff --git a/pkg/app/server.go b/pkg/app/server.go index 98bb11df01..b1cfb45f3a 100644 --- a/pkg/app/server.go +++ b/pkg/app/server.go @@ -266,6 +266,7 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache) storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc()) + storeBuilder.WithGenerateMetadataOnlyStoresFunc(storeBuilder.DefaultGenerateMetadataOnlyStoresFunc()) proc.StartReaper() storeBuilder.WithUtilOptions(opts) @@ -275,6 +276,12 @@ func RunKubeStateMetrics(ctx context.Context, opts *options.Options) error { } storeBuilder.WithKubeClient(kubeClient) + metadataOnlyKubeClient, err := util.CreateMetadataOnlyKubeClient(opts.Apiserver, opts.Kubeconfig) + if err != nil { + return fmt.Errorf("failed to create metadata-only client: %v", err) + } + storeBuilder.WithMetadataOnlyKubeClient(metadataOnlyKubeClient) + storeBuilder.WithSharding(opts.Shard, opts.TotalShards) if err := storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList); err != nil { return fmt.Errorf("failed to set up annotations allowlist: %v", err) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 43a09b57e0..dfa79c8aea 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/client_golang/prometheus" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" "k8s.io/client-go/tools/cache" internalstore "k8s.io/kube-state-metrics/v2/internal/store" @@ -84,6 +85,11 @@ func (b *Builder) WithKubeClient(c clientset.Interface) { b.internal.WithKubeClient(c) } +// WithMetadataOnlyKubeClient sets the metadataOnlyKubeClient property of a Builder. +func (b *Builder) WithMetadataOnlyKubeClient(c metadata.Interface) { + b.internal.WithMetadataOnlyKubeClient(c) +} + // WithCustomResourceClients sets the customResourceClients property of a Builder. func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) { b.internal.WithCustomResourceClients(cs) @@ -120,6 +126,11 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc { return b.internal.DefaultGenerateStoresFunc() } +// DefaultGenerateMetadataOnlyStoresFunc returns default buildMetadataOnlyStore function +func (b *Builder) DefaultGenerateMetadataOnlyStoresFunc() ksmtypes.BuildMetadataOnlyStoresFunc { + return b.internal.DefaultGenerateMetadataOnlyStoresFunc() +} + // DefaultGenerateCustomResourceStoresFunc returns default buildStores function func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc { return b.internal.DefaultGenerateCustomResourceStoresFunc() diff --git a/pkg/builder/types/interfaces.go b/pkg/builder/types/interfaces.go index b7ba595da1..81388d6c6b 100644 --- a/pkg/builder/types/interfaces.go +++ b/pkg/builder/types/interfaces.go @@ -23,6 +23,7 @@ import ( "github.com/prometheus/client_golang/prometheus" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" "k8s.io/client-go/tools/cache" "k8s.io/kube-state-metrics/v2/pkg/customresource" @@ -38,6 +39,7 @@ type BuilderInterface interface { WithFieldSelectorFilter(fieldSelectors string) WithSharding(shard int32, totalShards int) WithContext(ctx context.Context) + WithMetadataOnlyKubeClient(c metadata.Interface) WithKubeClient(c clientset.Interface) WithCustomResourceClients(cs map[string]interface{}) WithUsingAPIServerCache(u bool) @@ -46,6 +48,8 @@ type BuilderInterface interface { WithAllowLabels(l map[string][]string) error WithGenerateStoresFunc(f BuildStoresFunc) DefaultGenerateStoresFunc() BuildStoresFunc + WithGenerateMetadataOnlyStoresFunc(f BuildMetadataOnlyStoresFunc) + DefaultGenerateMetadataOnlyStoresFunc() BuildMetadataOnlyStoresFunc DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) Build() metricsstore.MetricsWriterList @@ -60,6 +64,13 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator, useAPIServerCache bool, ) []cache.Store +// BuildMetadataOnlyStoresFunc function signature that is used to return a list of cache.Store +type BuildMetadataOnlyStoresFunc func(metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(kubeClient metadata.Interface, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store + // BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store type BuildCustomResourceStoresFunc func(resourceName string, metricFamilies []generator.FamilyGenerator, diff --git a/pkg/util/utils.go b/pkg/util/utils.go index e10afdcd25..de1090bf14 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -25,6 +25,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/discovery" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" @@ -38,6 +39,7 @@ import ( var config *rest.Config var currentKubeClient clientset.Interface +var currentMetadataOnlyKubeClient metadata.Interface var currentDiscoveryClient *discovery.DiscoveryClient // CreateKubeClient creates a Kubernetes clientset and a custom resource clientset. @@ -79,6 +81,34 @@ func CreateKubeClient(apiserver string, kubeconfig string) (clientset.Interface, return kubeClient, nil } +// CreateMetadataOnlyKubeClient creates a Kubernetes clientset and a custom resource clientset. +func CreateMetadataOnlyKubeClient(apiserver string, kubeconfig string) (metadata.Interface, error) { + if currentMetadataOnlyKubeClient != nil { + return currentMetadataOnlyKubeClient, nil + } + + var err error + + if config == nil { + var err error + config, err = clientcmd.BuildConfigFromFlags(apiserver, kubeconfig) + if err != nil { + return nil, err + } + } + config.UserAgent = fmt.Sprintf("%s/%s (%s/%s) kubernetes/%s", "kube-state-metrics (metadataonly)", version.Version, runtime.GOOS, runtime.GOARCH, version.Revision) + config.AcceptContentTypes = "application/vnd.kubernetes.protobuf;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1,application/json;as=PartialObjectMetadataList;g=meta.k8s.io;v=v1,application/json,application/vnd.kubernetes.protobuf;as=PartialObjectMetadata;g=meta.k8s.io;v=v1,application/json;as=PartialObjectMetadata;g=meta.k8s.io;v=v1" + config.ContentType = "application/vnd.kubernetes.protobuf" + + kubeClient, err := metadata.NewForConfig(config) + if err != nil { + return nil, err + } + + currentMetadataOnlyKubeClient = kubeClient + return kubeClient, nil +} + // CreateCustomResourceClients creates a custom resource clientset. func CreateCustomResourceClients(apiserver string, kubeconfig string, factories ...customresource.RegistryFactory) (map[string]interface{}, error) { // Not relying on memoized clients here because the factories are subject to change.