diff --git a/pkg/config/config.go b/pkg/config/config.go index 0dd19208..0bc14a84 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -26,37 +26,39 @@ import ( // Out of box defaults const ( - COLLECTOR_API_VERSION = "2.11.0" - DEFAULT_AGGREGATOR_URL = "https://localhost:3010" // this will be deprecated in the future - DEFAULT_AGGREGATOR_HOST = "https://localhost" - DEFAULT_AGGREGATOR_PORT = "3010" - DEFAULT_CLUSTER_NAME = "local-cluster" - DEFAULT_POD_NAMESPACE = "open-cluster-management" - DEFAULT_HEARTBEAT_MS = 300000 // 5 min - DEFAULT_MAX_BACKOFF_MS = 600000 // 10 min - DEFAULT_REDISCOVER_RATE_MS = 120000 // 2 min - DEFAULT_REPORT_RATE_MS = 5000 // 5 seconds - DEFAULT_RETRY_JITTER_MS = 5000 // 5 seconds - DEFAULT_RUNTIME_MODE = "production" + COLLECTOR_API_VERSION = "2.11.0" + DEFAULT_AGGREGATOR_URL = "https://localhost:3010" // this will be deprecated in the future + DEFAULT_AGGREGATOR_HOST = "https://localhost" + DEFAULT_AGGREGATOR_PORT = "3010" + DEFAULT_COLLECT_ANNOTATIONS = false + DEFAULT_CLUSTER_NAME = "local-cluster" + DEFAULT_POD_NAMESPACE = "open-cluster-management" + DEFAULT_HEARTBEAT_MS = 300000 // 5 min + DEFAULT_MAX_BACKOFF_MS = 600000 // 10 min + DEFAULT_REDISCOVER_RATE_MS = 120000 // 2 min + DEFAULT_REPORT_RATE_MS = 5000 // 5 seconds + DEFAULT_RETRY_JITTER_MS = 5000 // 5 seconds + DEFAULT_RUNTIME_MODE = "production" ) // Configuration options for the search-collector. type Config struct { AggregatorConfig *rest.Config // Config object for hub. Used to get TLS credentials. - AggregatorConfigFile string `env:"HUB_CONFIG"` // Config file for hub. Will be mounted in a secret. - AggregatorURL string `env:"AGGREGATOR_URL"` // URL of the Aggregator, includes port but not any path - AggregatorHost string `env:"AGGREGATOR_HOST"` // Host of the Aggregator - AggregatorPort string `env:"AGGREGATOR_PORT"` // Port of the Aggregator - ClusterName string `env:"CLUSTER_NAME"` // The name of of the cluster where this pod is running - PodNamespace string `env:"POD_NAMESPACE"` // The namespace of this pod - DeployedInHub bool `env:"DEPLOYED_IN_HUB"` // Tracks if deployed in the Hub or Managed cluster - HeartbeatMS int `env:"HEARTBEAT_MS"` // Interval(ms) to send empty payload to ensure connection - KubeConfig string `env:"KUBECONFIG"` // Local kubeconfig path - MaxBackoffMS int `env:"MAX_BACKOFF_MS"` // Maximum backoff in ms to wait after error - RediscoverRateMS int `env:"REDISCOVER_RATE_MS"` // Interval(ms) to poll for changes to CRDs - RetryJitterMS int `env:"RETRY_JITTER_MS"` // Random jitter added to backoff wait. - ReportRateMS int `env:"REPORT_RATE_MS"` // Interval(ms) to send changes to the aggregator - RuntimeMode string `env:"RUNTIME_MODE"` // Running mode (development or production) + AggregatorConfigFile string `env:"HUB_CONFIG"` // Config file for hub. Will be mounted in a secret. + AggregatorURL string `env:"AGGREGATOR_URL"` // URL of the Aggregator, includes port but not any path + AggregatorHost string `env:"AGGREGATOR_HOST"` // Host of the Aggregator + AggregatorPort string `env:"AGGREGATOR_PORT"` // Port of the Aggregator + CollectAnnotations bool `env:"COLLECT_ANNOTATIONS"` // Collect all annotations with values <=64 characters + ClusterName string `env:"CLUSTER_NAME"` // The name of of the cluster where this pod is running + PodNamespace string `env:"POD_NAMESPACE"` // The namespace of this pod + DeployedInHub bool `env:"DEPLOYED_IN_HUB"` // Tracks if deployed in the Hub or Managed cluster + HeartbeatMS int `env:"HEARTBEAT_MS"` // Interval(ms) to send empty payload to ensure connection + KubeConfig string `env:"KUBECONFIG"` // Local kubeconfig path + MaxBackoffMS int `env:"MAX_BACKOFF_MS"` // Maximum backoff in ms to wait after error + RediscoverRateMS int `env:"REDISCOVER_RATE_MS"` // Interval(ms) to poll for changes to CRDs + RetryJitterMS int `env:"RETRY_JITTER_MS"` // Random jitter added to backoff wait. + ReportRateMS int `env:"REPORT_RATE_MS"` // Interval(ms) to send changes to the aggregator + RuntimeMode string `env:"RUNTIME_MODE"` // Running mode (development or production) } var Cfg = Config{} @@ -112,6 +114,16 @@ func InitConfig() { } setDefault(&Cfg.KubeConfig, "KUBECONFIG", defaultKubePath) + if collectAnnotations := os.Getenv("COLLECT_ANNOTATIONS"); collectAnnotations != "" { + glog.Infof("Using COLLECT_ANNOTATIONS from environment: %s", collectAnnotations) + + var err error + Cfg.CollectAnnotations, err = strconv.ParseBool(collectAnnotations) + if err != nil { + glog.Errorf("Error parsing env COLLECT_ANNOTATIONS, defaulting to false: %v", err) + } + } + // Special logic for setting DEPLOYED_IN_HUB with default to false if val := os.Getenv("DEPLOYED_IN_HUB"); val != "" { glog.Infof("Using DEPLOYED_IN_HUB from environment: %s", val) diff --git a/pkg/transforms/common.go b/pkg/transforms/common.go index 57d97c0c..fe6e286b 100644 --- a/pkg/transforms/common.go +++ b/pkg/transforms/common.go @@ -13,10 +13,12 @@ package transforms import ( "strings" "time" + "unicode/utf8" "github.com/golang/glog" "github.com/stolostron/search-collector/pkg/config" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" apiTypes "k8s.io/apimachinery/pkg/types" ) @@ -27,6 +29,43 @@ type NodeStore struct { ByKindNamespaceName map[string]map[string]map[string]Node } +// commonAnnotations returns the annotations with values <= 64 characters. It also removes the +// last-applied-configuration annotation regardless of length. +func commonAnnotations(object v1.Object) map[string]string { + // If CollectAnnotations is not true, then only collect annotations for allow listed resources. + if !config.Cfg.CollectAnnotations { + typeInfo, ok := object.(v1.Type) + if !ok { + return nil + } + + gv, err := schema.ParseGroupVersion(typeInfo.GetAPIVersion()) + if err != nil { + return nil + } + + switch gv.Group { + case "policies.open-cluster-management.io": + case "constraints.gatekeeper.sh": + default: + return nil + } + } + + annotations := object.GetAnnotations() + + // This annotation is large and useless + delete(annotations, "kubectl.kubernetes.io/last-applied-configuration") + + for key, val := range annotations { + if utf8.RuneCountInString(val) > 64 { + delete(annotations, key) + } + } + + return annotations +} + // Extracts the common properties from a k8s resource of any type and returns a map ready to be put in a Node func commonProperties(resource v1.Object) map[string]interface{} { ret := make(map[string]interface{}) @@ -40,6 +79,13 @@ func commonProperties(resource v1.Object) map[string]interface{} { if resource.GetLabels() != nil { ret["label"] = resource.GetLabels() } + + annotations := commonAnnotations(resource) + + if annotations != nil { + ret["annotation"] = annotations + } + if resource.GetNamespace() != "" { ret["namespace"] = resource.GetNamespace() } diff --git a/pkg/transforms/common_test.go b/pkg/transforms/common_test.go index bf938980..4588cf2e 100644 --- a/pkg/transforms/common_test.go +++ b/pkg/transforms/common_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/stolostron/search-collector/pkg/config" v1 "k8s.io/api/core/v1" machineryV1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -32,10 +33,16 @@ func CreateGenericResource() machineryV1.Object { p.UID = "00aa0000-00aa-00a0-a000-00000a00a0a0" p.CreationTimestamp = timestamp p.Labels = labels + p.Annotations = map[string]string{"hello": "world"} return &p } func TestCommonProperties(t *testing.T) { + config.Cfg.CollectAnnotations = true + + defer func() { + config.Cfg.CollectAnnotations = false + }() res := CreateGenericResource() timeString := timestamp.UTC().Format(time.RFC3339) @@ -46,6 +53,7 @@ func TestCommonProperties(t *testing.T) { AssertEqual("name", cp["name"], interface{}("testpod"), t) AssertEqual("namespace", cp["namespace"], interface{}("default"), t) AssertEqual("created", cp["created"], interface{}(timeString), t) + AssertEqual("annotation", cp["annotation"].(map[string]string)["hello"], "world", t) noLabels := true for key, value := range cp["label"].(map[string]string) { diff --git a/pkg/transforms/genericResource_test.go b/pkg/transforms/genericResource_test.go index 9adc9b4e..c7405f85 100644 --- a/pkg/transforms/genericResource_test.go +++ b/pkg/transforms/genericResource_test.go @@ -5,10 +5,20 @@ package transforms import ( "testing" + "github.com/stolostron/search-collector/pkg/config" + "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" ) func Test_genericResourceFromConfig(t *testing.T) { + config.Cfg.CollectAnnotations = true + + defer func() { + config.Cfg.CollectAnnotations = false + }() + var r unstructured.Unstructured UnmarshalFile("clusterserviceversion.json", &r, t) node := GenericResourceBuilder(&r).BuildNode() @@ -19,11 +29,59 @@ func Test_genericResourceFromConfig(t *testing.T) { AssertEqual("namespace", node.Properties["namespace"], "open-cluster-management", t) AssertEqual("created", node.Properties["created"], "2023-08-23T15:54:22Z", t) + annotations, ok := node.Properties["annotation"].(map[string]string) + assert.True(t, ok) + + // Ensure last-applied-configuration and other large annotations are not present + expectedAnnotationKeys := sets.New[string]( + "capabilities", "categories", "certified", "createdAt", "olm.operatorGroup", + "olm.operatorNamespace", "olm.targetNamespaces", "operatorframework.io/suggested-namespace", + "operators.openshift.io/infrastructure-features", "operators.operatorframework.io/internal-objects", "support", + ) + + actualAnnotationKeys := sets.Set[string]{} + + for key := range annotations { + actualAnnotationKeys.Insert(key) + } + + assert.True(t, expectedAnnotationKeys.Equal(actualAnnotationKeys)) + // Verify properties defined in the transform config AssertEqual("display", node.Properties["display"], "Advanced Cluster Management for Kubernetes", t) AssertEqual("phase", node.Properties["phase"], "Succeeded", t) AssertEqual("version", node.Properties["version"], "2.9.0", t) + // Verify that annotations are not collected when COLLECT_ANNOTATIONS is false + config.Cfg.CollectAnnotations = false + + node = GenericResourceBuilder(&r).BuildNode() + assert.Nil(t, node.Properties["annotations"]) +} + +func Test_allowListedForAnnotations(t *testing.T) { + obj := unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "policies.open-cluster-management.io", Kind: "Policy", Version: "v1", + }) + obj.SetAnnotations(map[string]string{"hello": "world"}) + + node := GenericResourceBuilder(&obj).BuildNode() + assert.NotNil(t, node.Properties["annotation"]) + + obj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "constraints.gatekeeper.sh", Kind: "K8sRequiredLabels", Version: "v1beta1", + }) + + node = GenericResourceBuilder(&obj).BuildNode() + assert.NotNil(t, node.Properties["annotation"]) + + obj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "something.domain.example", Kind: "SomeKind", Version: "v1", + }) + + node = GenericResourceBuilder(&obj).BuildNode() + assert.Nil(t, node.Properties["annotation"]) } func Test_genericResourceFromConfigVM(t *testing.T) { diff --git a/pkg/transforms/genericresource.go b/pkg/transforms/genericresource.go index c752409e..dedd23a7 100644 --- a/pkg/transforms/genericresource.go +++ b/pkg/transforms/genericresource.go @@ -99,6 +99,13 @@ func genericProperties(r *unstructured.Unstructured) map[string]interface{} { if r.GetLabels() != nil { ret["label"] = r.GetLabels() } + + annotations := commonAnnotations(r) + + if annotations != nil { + ret["annotation"] = annotations + } + if r.GetNamespace() != "" { ret["namespace"] = r.GetNamespace() } diff --git a/test-data/clusterserviceversion.json b/test-data/clusterserviceversion.json index 4cc4b2e0..c74927d8 100644 --- a/test-data/clusterserviceversion.json +++ b/test-data/clusterserviceversion.json @@ -18,6 +18,7 @@ "operators.openshift.io/infrastructure-features": "[\"disconnected\", \"proxy-aware\", \"fips\"]", "operators.openshift.io/valid-subscription": "[\"OpenShift Platform Plus\", \"Red Hat Advanced Cluster Management for Kubernetes\"]", "operators.operatorframework.io/internal-objects": "[]", + "kubectl.kubernetes.io/last-applied-configuration": "something", "support": "Red Hat" }, "creationTimestamp": "2023-08-23T15:54:22Z",