diff --git a/cicd-scripts/setup-e2e-tests.sh b/cicd-scripts/setup-e2e-tests.sh index 6e002da29..0d7b215db 100755 --- a/cicd-scripts/setup-e2e-tests.sh +++ b/cicd-scripts/setup-e2e-tests.sh @@ -152,7 +152,7 @@ deploy_mco_operator() { cd ${ROOTDIR}/operators/multiclusterobservability/config/manager && kustomize edit set image quay.io/stolostron/multicluster-observability-operator="${IMAGE_REPO}/multicluster-observability-operator:${LATEST_SNAPSHOT}" fi cd ${ROOTDIR} - kustomize build ${ROOTDIR}/operators/multiclusterobservability/config/default | kubectl apply -n ${OCM_DEFAULT_NS} --server-side=true -f - + kustomize build ${ROOTDIR}/operators/multiclusterobservability/config/default | kubectl apply -n ${OCM_DEFAULT_NS} --server-side=true --force-conflicts -f - # wait until mco is ready wait_for_deployment_ready 10 60s ${OCM_DEFAULT_NS} multicluster-observability-operator diff --git a/loaders/dashboards/Dockerfile b/loaders/dashboards/Dockerfile index 418254cae..0d6bf777c 100644 --- a/loaders/dashboards/Dockerfile +++ b/loaders/dashboards/Dockerfile @@ -1,6 +1,6 @@ # Copyright Contributors to the Open Cluster Management project -FROM registry.ci.openshift.org/stolostron/builder:go1.21-linux AS builder +FROM registry.ci.openshift.org/stolostron/builder:go1.22-linux AS builder WORKDIR /workspace COPY go.sum go.mod ./loaders/dashboards ./ diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go index a4d0c96d1..ab621998e 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go @@ -13,6 +13,9 @@ import ( "strings" "time" + imagev1 "github.com/openshift/api/image/v1" + imagev1client "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" + operatorconfig "github.com/stolostron/multicluster-observability-operator/operators/pkg/config" addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" @@ -80,13 +83,14 @@ var ( // MultiClusterObservabilityReconciler reconciles a MultiClusterObservability object type MultiClusterObservabilityReconciler struct { - Manager manager.Manager - Client client.Client - Log logr.Logger - Scheme *runtime.Scheme - CRDMap map[string]bool - APIReader client.Reader - RESTMapper meta.RESTMapper + Manager manager.Manager + Client client.Client + Log logr.Logger + Scheme *runtime.Scheme + CRDMap map[string]bool + APIReader client.Reader + RESTMapper meta.RESTMapper + ImageClient *imagev1client.ImageV1Client } // +kubebuilder:rbac:groups=observability.open-cluster-management.io,resources=multiclusterobservabilities,verbs=get;list;watch;create;update;patch;delete @@ -250,8 +254,9 @@ func (r *MultiClusterObservabilityReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, err } instance.Spec.StorageConfig.StorageClass = storageClassSelected + // Render the templates with a specified CR - renderer := rendering.NewMCORenderer(instance, r.Client) + renderer := rendering.NewMCORenderer(instance, r.Client, r.ImageClient) toDeploy, err := renderer.Render() if err != nil { reqLogger.Error(err, "Failed to render multiClusterMonitoring templates") @@ -448,7 +453,6 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager) cmPred := GetConfigMapPredicateFunc() secretPred := GetAlertManagerSecretPredicateFunc() namespacePred := GetNamespacePredicateFunc() - ctrBuilder := ctrl.NewControllerManagedBy(mgr). // Watch for changes to primary resource MultiClusterObservability with predicate For(&mcov1beta2.MultiClusterObservability{}, builder.WithPredicates(mcoPred)). @@ -489,6 +493,23 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager) return nil }), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{})) + if _, err := mgr.GetRESTMapper().KindFor(schema.GroupVersionResource{ + Group: "image.openshift.io", + Version: "v1", + Resource: "imagestreams", + }); err != nil { + if meta.IsNoMatchError(err) { + log.Info("image.openshift.io/v1/imagestreams is not available") + } else { + log.Error(err, "failed to get kind for image.openshift.io/v1/imagestreams") + os.Exit(1) + } + } else { + // Images stream is only available in OpenShift + imageStreamPred := GetImageStreamPredicateFunc() + ctrBuilder = ctrBuilder.Watches(&imagev1.ImageStream{}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(imageStreamPred)) + } + mchGroupKind := schema.GroupKind{Group: mchv1.GroupVersion.Group, Kind: "MultiClusterHub"} if _, err := r.RESTMapper.RESTMapping(mchGroupKind, mchv1.GroupVersion.Version); err == nil { mchPred := GetMCHPredicateFunc(c) diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/predicate_func.go b/operators/multiclusterobservability/controllers/multiclusterobservability/predicate_func.go index 9f195f228..bd2f27f10 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/predicate_func.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/predicate_func.go @@ -189,3 +189,20 @@ func GetNamespacePredicateFunc() predicate.Funcs { }, } } + +func GetImageStreamPredicateFunc() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return e.Object.GetName() == config.OauthProxyImageStreamName + }, + UpdateFunc: func(e event.UpdateEvent) bool { + if e.ObjectNew.GetName() != config.OauthProxyImageStreamName { + return false + } + return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + } +} diff --git a/operators/multiclusterobservability/main.go b/operators/multiclusterobservability/main.go index 69e65c8f6..58defaa3d 100644 --- a/operators/multiclusterobservability/main.go +++ b/operators/multiclusterobservability/main.go @@ -10,6 +10,9 @@ import ( "fmt" "os" + imagev1 "github.com/openshift/api/image/v1" + + imagev1client "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -69,6 +72,7 @@ func init() { utilruntime.Must(observatoriumAPIs.AddToScheme(scheme)) utilruntime.Must(prometheusv1.AddToScheme(scheme)) utilruntime.Must(addonv1alpha1.AddToScheme(scheme)) + utilruntime.Must(imagev1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -282,14 +286,21 @@ func main() { config.MCGHCrdName: mcghCrdExists, } + imageClient, err := imagev1client.NewForConfig(ctrl.GetConfigOrDie()) + if err != nil { + setupLog.Error(err, "failed to create openshift image client") + os.Exit(1) + } + if err = (&mcoctrl.MultiClusterObservabilityReconciler{ - Manager: mgr, - Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("MultiClusterObservability"), - Scheme: mgr.GetScheme(), - CRDMap: crdMaps, - APIReader: mgr.GetAPIReader(), - RESTMapper: mgr.GetRESTMapper(), + Manager: mgr, + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("MultiClusterObservability"), + Scheme: mgr.GetScheme(), + CRDMap: crdMaps, + APIReader: mgr.GetAPIReader(), + RESTMapper: mgr.GetRESTMapper(), + ImageClient: imageClient, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "MultiClusterObservability") os.Exit(1) diff --git a/operators/multiclusterobservability/pkg/config/config.go b/operators/multiclusterobservability/pkg/config/config.go index aab3ed945..fd89878c2 100644 --- a/operators/multiclusterobservability/pkg/config/config.go +++ b/operators/multiclusterobservability/pkg/config/config.go @@ -9,9 +9,12 @@ import ( "fmt" "net/url" "os" + "reflect" "strings" "time" + imagev1client "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" + ocinfrav1 "github.com/openshift/api/config/v1" operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" @@ -148,11 +151,6 @@ const ( KubeRBACProxyKey = "kube_rbac_proxy" KubeRBACProxyImgName = "kube-rbac-proxy" - OauthProxyImgRepo = "quay.io/stolostron" - OauthProxyImgName = "origin-oauth-proxy" - OauthProxyImgTagSuffix = "2.0.12-SNAPSHOT-2021-06-11-19-40-10" - OauthProxyKey = "oauth_proxy" - EndpointControllerImgName = "endpoint-monitoring-operator" EndpointControllerKey = "endpoint_monitoring_operator" @@ -219,6 +217,11 @@ const ( HubEndpointSaName = "endpoint-observability-operator-sa" ) +const ( + OauthProxyImageStreamName = "oauth-proxy" + OauthProxyImageStreamNamespace = "openshift" +) + // ObjectStorgeConf is used to Unmarshal from bytes to do validation. type ObjectStorgeConf struct { Type string `yaml:"type"` @@ -828,3 +831,27 @@ func IsAlertingDisabledInSpec(mco *observabilityv1beta2.MultiClusterObservabilit annotations := mco.GetAnnotations() return annotations != nil && annotations[AnnotationDisableMCOAlerting] == "true" } + +func GetOauthProxyImage(imageClient imagev1client.ImageV1Interface) (bool, string) { + if imageClient != nil && !reflect.ValueOf(imageClient).IsNil() { + // set oauth-proxy from imagestream.image.openshift.io + oauthImageStream, err := imageClient.ImageStreams(OauthProxyImageStreamNamespace). + Get(context.TODO(), OauthProxyImageStreamName, v1.GetOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + return false, "" + } + // do not expect error = IsNotFound in OCP environment. + // But for e2e test, it can be. for this case, just ignore + } else { + if oauthImageStream.Spec.Tags != nil { + tag := oauthImageStream.Spec.Tags[0] + if tag.From != nil && tag.From.Kind == "DockerImage" && len(tag.From.Name) > 0 { + return true, tag.From.Name + } + } + } + } + return false, "" + +} diff --git a/operators/multiclusterobservability/pkg/rendering/renderer.go b/operators/multiclusterobservability/pkg/rendering/renderer.go index 0605a72ca..b66cf28cf 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer.go @@ -5,6 +5,7 @@ package rendering import ( + imagev1client "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -24,6 +25,7 @@ var log = logf.Log.WithName("renderer") type MCORenderer struct { kubeClient client.Client + imageClient *imagev1client.ImageV1Client renderer *rendererutil.Renderer cr *obv1beta2.MultiClusterObservability renderGrafanaFns map[string]rendererutil.RenderFn @@ -33,11 +35,12 @@ type MCORenderer struct { renderMCOAFns map[string]rendererutil.RenderFn } -func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservability, kubeClient client.Client) *MCORenderer { +func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservability, kubeClient client.Client, imageClient *imagev1client.ImageV1Client) *MCORenderer { mcoRenderer := &MCORenderer{ - renderer: rendererutil.NewRenderer(), - cr: multipleClusterMonitoring, - kubeClient: kubeClient, + renderer: rendererutil.NewRenderer(), + cr: multipleClusterMonitoring, + kubeClient: kubeClient, + imageClient: imageClient, } mcoRenderer.newGranfanaRenderer() mcoRenderer.newAlertManagerRenderer() diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go index 188396eaa..d490488de 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager.go @@ -100,10 +100,8 @@ func (r *MCORenderer) renderAlertManagerStatefulSet(res *resource.Resource, if found { spec.Containers[1].Image = image } - // the oauth-proxy image only exists in mch-image-manifest configmap - // pass nil annotation to make sure oauth-proxy overrided from mch-image-manifest - found, image = mcoconfig.ReplaceImage(nil, mcoconfig.OauthProxyImgRepo, - mcoconfig.OauthProxyKey) + + found, image = mcoconfig.GetOauthProxyImage(r.imageClient) if found { spec.Containers[2].Image = image } diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go index c04c4de91..959517a6b 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_alertmanager_test.go @@ -40,10 +40,9 @@ func TestAlertManagerRenderer(t *testing.T) { } containerNameToMchKey := map[string]string{ - "alertmanager": "prometheus_alertmanager", - "config-reloader": "configmap_reloader", - "alertmanager-proxy": "oauth_proxy", - "kube-rbac-proxy": "kube_rbac_proxy", + "alertmanager": "prometheus_alertmanager", + "config-reloader": "configmap_reloader", + "kube-rbac-proxy": "kube_rbac_proxy", } mchImageManifest := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -57,12 +56,12 @@ func TestAlertManagerRenderer(t *testing.T) { Data: map[string]string{ "prometheus_alertmanager": "quay.io/rhacm2/alertmanager:latest", "configmap_reloader": "quay.io/rhacm2/configmap-reloader:latest", - "oauth_proxy": "quay.io/rhacm2/oauth_proxy:latest", "kube_rbac_proxy": "quay.io/rhacm2/kube-rbac-proxy:latest", }, } kubeClient := fake.NewClientBuilder().WithObjects(clientCa, mchImageManifest).Build() + alertResources := renderTemplates(t, kubeClient, makeBaseMco()) // clientCa configmap must be filled with the client-ca-file data @@ -75,6 +74,11 @@ func TestAlertManagerRenderer(t *testing.T) { sts := &appsv1.StatefulSet{} runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, sts) for _, container := range sts.Spec.Template.Spec.Containers { + // oauth-proxy container is not in the mch-image-manifest configmap + // we use image-streams to get image for oauth-proxy + if container.Name == "alertmanager-proxy" { + continue + } assert.Equal(t, mchImageManifest.Data[containerNameToMchKey[container.Name]], container.Image) } } @@ -237,7 +241,7 @@ func renderTemplates(t *testing.T, kubeClient client.Client, mco *mcov1beta2.Mul defer os.Unsetenv(templatesutil.TemplatesPathEnvVar) config.ReadImageManifestConfigMap(kubeClient, "v1") - renderer := NewMCORenderer(mco, kubeClient) + renderer := NewMCORenderer(mco, kubeClient, nil) //load and render alertmanager templates alertTemplates, err := templates.GetOrLoadAlertManagerTemplates(templatesutil.GetTemplateRenderer()) diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_grafana.go b/operators/multiclusterobservability/pkg/rendering/renderer_grafana.go index 36962fea5..d60e8af23 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_grafana.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_grafana.go @@ -68,8 +68,7 @@ func (r *MCORenderer) renderGrafanaDeployments(res *resource.Resource, } spec.Containers[1].ImagePullPolicy = imagePullPolicy - found, image = config.ReplaceImage(nil, config.OauthProxyImgRepo, - config.OauthProxyKey) + found, image = config.GetOauthProxyImage(r.imageClient) if found { spec.Containers[2].Image = image } diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_proxy.go b/operators/multiclusterobservability/pkg/rendering/renderer_proxy.go index 3416639c1..4c9ddf3e5 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_proxy.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_proxy.go @@ -93,10 +93,7 @@ func (r *MCORenderer) renderProxyDeployment(res *resource.Resource, spec.Containers[0].Image = image } - // the oauth-proxy image only exists in mch-image-manifest configmap - // pass nil annotation to make sure oauth-proxy overrided from mch-image-manifest - found, image = mcoconfig.ReplaceImage(nil, mcoconfig.OauthProxyImgRepo, - mcoconfig.OauthProxyKey) + found, image = mcoconfig.GetOauthProxyImage(r.imageClient) if found { spec.Containers[1].Image = image } diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_test.go b/operators/multiclusterobservability/pkg/rendering/renderer_test.go index 50acac4f1..313091832 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_test.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_test.go @@ -5,10 +5,17 @@ package rendering import ( + "context" "os" "path" "testing" + imagev1 "github.com/openshift/api/image/v1" + fakeimageclient "github.com/openshift/client-go/image/clientset/versioned/fake" + fakeimagev1client "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1/fake" + "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/config" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -59,9 +66,39 @@ func TestRender(t *testing.T) { } kubeClient := fake.NewClientBuilder().WithObjects(clientCa).Build() - renderer := NewMCORenderer(mchcr, kubeClient) + renderer := NewMCORenderer(mchcr, kubeClient, nil) _, err = renderer.Render() if err != nil { t.Fatalf("failed to render MultiClusterObservability: %v", err) } } + +func TestGetOauthProxyFromImageStreams(t *testing.T) { + imageClient := &fakeimagev1client.FakeImageV1{Fake: &(fakeimageclient.NewSimpleClientset().Fake)} + _, err := imageClient.ImageStreams(config.OauthProxyImageStreamNamespace).Create(context.Background(), + &imagev1.ImageStream{ + ObjectMeta: metav1.ObjectMeta{ + Name: config.OauthProxyImageStreamName, + Namespace: config.OauthProxyImageStreamNamespace, + }, + Spec: imagev1.ImageStreamSpec{ + Tags: []imagev1.TagReference{ + { + Name: "v4.4", + From: &corev1.ObjectReference{ + Kind: "DockerImage", + Name: "quay.io/openshift-release-dev/ocp-v4.0-art-dev", + }, + }, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + found, oauthProxyImage := config.GetOauthProxyImage(imageClient) + if !found { + t.Fatal("Failed to get oauth proxy image") + } + assert.Equal(t, "quay.io/openshift-release-dev/ocp-v4.0-art-dev", oauthProxyImage) +} diff --git a/proxy/Dockerfile b/proxy/Dockerfile index 9ded59749..2122c99e4 100644 --- a/proxy/Dockerfile +++ b/proxy/Dockerfile @@ -1,6 +1,6 @@ # Copyright Contributors to the Open Cluster Management project -FROM registry.ci.openshift.org/stolostron/builder:go1.21-linux AS builder +FROM registry.ci.openshift.org/stolostron/builder:go1.22-linux AS builder WORKDIR /workspace COPY go.sum go.mod ./ diff --git a/proxy/cmd/main.go b/proxy/cmd/main.go index 9e3e2bb8f..0960c7f78 100644 --- a/proxy/cmd/main.go +++ b/proxy/cmd/main.go @@ -82,7 +82,7 @@ func main() { } if err := util.InitAccessReviewer(kubeConfig); err != nil { - klog.Fatalf("failed to Initialize Access Reviewer: %v", err) + klog.Fatalf("failed to Initialize Access Reviewer: %v test", err) } // watch all managed clusters