Merge pull request #296 from gianlucam76/main

Merge dev to main

Makefile

@@ -220,6 +220,9 @@ create-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create a new kin
 	@echo wait for calico pod
 	$(KUBECTL) --kubeconfig=./test/fv/workload_kubeconfig wait --for=condition=Available deployment/calico-kube-controllers -n kube-system --timeout=$(TIMEOUT)
 
+	@echo apply reloader CRD to managed cluster
+	$(KUBECTL) --kubeconfig=./test/fv/workload_kubeconfig apply -f https://raw.githubusercontent.com/projectsveltos/libsveltos/$(TAG)/config/crd/bases/lib.projectsveltos.io_reloaders.yaml
+
 .PHONY: delete-cluster
 delete-cluster: $(KIND) ## Deletes the kind cluster $(CONTROL_CLUSTER_NAME)
 	$(KIND) delete cluster --name $(CONTROL_CLUSTER_NAME)

api/v1alpha1/clusterprofile_types.go

@@ -251,6 +251,15 @@ type ClusterProfileSpec struct {
 	// +optional
 	StopMatchingBehavior StopMatchingBehavior `json:"stopMatchingBehavior,omitempty"`
 
+	// Reloader indicates whether Deployment/StatefulSet/DaemonSet instances deployed
+	// by Sveltos and part of this ClusterProfile need to be restarted via rolling upgrade
+	// when a ConfigMap/Secret instance mounted as volume is modified.
+	// When set to true, when any mounted ConfigMap/Secret is modified, Sveltos automatically
+	// starts a rolling upgrade for Deployment/StatefulSet/DaemonSet instances mounting it.
+	// +kubebuilder:default:=false
+	// +optional
+	Reloader bool `json:"reloader,omitempty"`
+
 	// TemplateResourceRefs is a list of resource to collect from the management cluster.
 	// Those resources' values will be used to instantiate templates contained in referenced
 	// PolicyRefs and Helm charts

config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml

@@ -246,6 +246,16 @@ spec:
                   - namespace
                   type: object
                 type: array
+              reloader:
+                default: false
+                description: Reloader indicates whether Deployment/StatefulSet/DaemonSet
+                  instances deployed by Sveltos and part of this ClusterProfile need
+                  to be restarted via rolling upgrade when a ConfigMap/Secret instance
+                  mounted as volume is modified. When set to true, when any mounted
+                  ConfigMap/Secret is modified, Sveltos automatically starts a rolling
+                  upgrade for Deployment/StatefulSet/DaemonSet instances mounting
+                  it.
+                type: boolean
               stopMatchingBehavior:
                 default: WithdrawPolicies
                 description: StopMatchingBehavior indicates what behavior should be

config/crd/bases/config.projectsveltos.io_clustersummaries.yaml

@@ -262,6 +262,16 @@ spec:
                       - namespace
                       type: object
                     type: array
+                  reloader:
+                    default: false
+                    description: Reloader indicates whether Deployment/StatefulSet/DaemonSet
+                      instances deployed by Sveltos and part of this ClusterProfile
+                      need to be restarted via rolling upgrade when a ConfigMap/Secret
+                      instance mounted as volume is modified. When set to true, when
+                      any mounted ConfigMap/Secret is modified, Sveltos automatically
+                      starts a rolling upgrade for Deployment/StatefulSet/DaemonSet
+                      instances mounting it.
+                    type: boolean
                   stopMatchingBehavior:
                     default: WithdrawPolicies
                     description: StopMatchingBehavior indicates what behavior should

controllers/clusterprofile_controller_test.go

@@ -40,6 +40,7 @@ import (
 
 	configv1alpha1 "github.com/projectsveltos/addon-controller/api/v1alpha1"
 	"github.com/projectsveltos/addon-controller/controllers"
+	"github.com/projectsveltos/addon-controller/internal/test/helpers/external"
 	"github.com/projectsveltos/addon-controller/pkg/scope"
 	libsveltosv1alpha1 "github.com/projectsveltos/libsveltos/api/v1alpha1"
 	libsveltosset "github.com/projectsveltos/libsveltos/lib/set"
@@ -93,7 +94,10 @@ var _ = Describe("ClusterProfile: Reconciler", func() {
 	})
 
 	It("getMatchingCluster considers both ClusterSelector and ClusterRefs", func() {
+		clusterCRD := external.TestClusterCRD.DeepCopy()
+
 		initObjects := []client.Object{
+			clusterCRD,
 			matchingCluster,
 			nonMatchingCluster,
 			clusterProfile,

controllers/controllers_suite_test.go

@@ -114,6 +114,12 @@ var _ = BeforeSuite(func() {
 	Expect(testEnv.Create(context.TODO(), addonComplianceCRD)).To(Succeed())
 	Expect(waitForObject(context.TODO(), testEnv, addonComplianceCRD)).To(Succeed())
 
+	var reloaderCRD *unstructured.Unstructured
+	reloaderCRD, err = utils.GetUnstructured(libsveltoscrd.GetReloaderCRDYAML())
+	Expect(err).To(BeNil())
+	Expect(testEnv.Create(context.TODO(), reloaderCRD)).To(Succeed())
+	Expect(waitForObject(context.TODO(), testEnv, reloaderCRD)).To(Succeed())
+
 	// Wait for synchronization
 	// Sometimes we otherwise get "no matches for kind "AddonCompliance" in version "lib.projectsveltos.io/v1alpha1"
 	time.Sleep(2 * time.Second)

controllers/export_test.go

@@ -133,3 +133,14 @@ var (
 	RunLuaValidations     = runLuaValidations
 	LuaValidation         = luaValidation
 )
+
+// reloader utils
+var (
+	WatchForRollingUpgrade                  = watchForRollingUpgrade
+	CreateReloaderInstance                  = createReloaderInstance
+	DeployReloaderInstance                  = deployReloaderInstance
+	RemoveReloaderInstance                  = removeReloaderInstance
+	UpdateReloaderWithDeployedResources     = updateReloaderWithDeployedResources
+	ConvertResourceReportsToObjectReference = convertResourceReportsToObjectReference
+	ConvertHelmResourcesToObjectReference   = convertHelmResourcesToObjectReference
+)

controllers/handlers_helm.go

@@ -129,15 +129,42 @@ func deployHelmCharts(ctx context.Context, c client.Client,
 		return err
 	}
 
+	var helmResources []libsveltosv1alpha1.HelmResources
+	if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1alpha1.SyncModeContinuousWithDriftDetection ||
+		clusterSummary.Spec.ClusterProfileSpec.Reloader {
+
+		helmResources, err = collectResourcesFromManagedHelmCharts(ctx, c, clusterSummary, kubeconfig, logger)
+		if err != nil {
+			return err
+		}
+	}
+
 	if clusterSummary.Spec.ClusterProfileSpec.SyncMode == configv1alpha1.SyncModeContinuousWithDriftDetection {
 		// Deploy resourceSummary
-		err = deployResourceSummaryWithHelmResources(ctx, c, clusterNamespace, clusterName,
-			clusterType, clusterSummary, kubeconfig, logger)
+		err = deployResourceSummaryInCluster(ctx, c, clusterNamespace, clusterName, clusterSummary.Name,
+			clusterType, nil, nil, helmResources, logger)
 		if err != nil {
-			return err
+			return nil
 		}
 	}
 
+	clusterProfileOwnerRef, err := configv1alpha1.GetClusterProfileOwnerReference(clusterSummary)
+	if err != nil {
+		return err
+	}
+
+	// Update Reloader instance. If ClusterProfile Reloader knob is set to true, sveltos will
+	// start a rolling upgrade for all Deployment/StatefulSet/DaemonSet instances deployed by Sveltos
+	// in the managed cluster when a mounted ConfigMap/Secret is updated. In order to do so, sveltos-agent
+	// needs to be instructed which Deployment/StatefulSet/DaemonSet instances require this behavior.
+	// Update corresponding Reloader instance (instance will be deleted if Reloader is set to false)
+	resources := convertHelmResourcesToObjectReference(helmResources)
+	err = updateReloaderWithDeployedResources(ctx, c, clusterProfileOwnerRef, configv1alpha1.FeatureHelm,
+		resources, clusterSummary, logger)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -161,6 +188,8 @@ func undeployHelmCharts(ctx context.Context, c client.Client,
 	logger = logger.WithValues("clusterSummary", clusterSummary.Name)
 	logger = logger.WithValues("admin", fmt.Sprintf("%s/%s", adminNamespace, adminName))
 
+	logger.V(logs.LogDebug).Info("undeployHelmCharts")
+
 	kubeconfigContent, err := clusterproxy.GetSecretData(ctx, c, clusterNamespace, clusterName,
 		adminNamespace, adminName, clusterSummary.Spec.ClusterType, logger)
 	if err != nil {
@@ -194,6 +223,13 @@ func undeployHelmCharts(ctx context.Context, c client.Client,
 	if err != nil {
 		return err
 	}
+
+	err = updateReloaderWithDeployedResources(ctx, c, clusterProfileOwnerRef, configv1alpha1.FeatureKustomize,
+		nil, clusterSummary, logger)
+	if err != nil {
+		return err
+	}
+
 	err = updateClusterConfiguration(ctx, c, clusterSummary, clusterProfileOwnerRef,
 		configv1alpha1.FeatureHelm, nil, []configv1alpha1.Chart{})
 	if err != nil {
@@ -280,6 +316,10 @@ func helmHash(ctx context.Context, c client.Client, clusterSummaryScope *scope.C
 	h := sha256.New()
 	var config string
 
+	// If Reloader changes, Reloader needs to be deployed or undeployed
+	// So consider it in the hash
+	config += fmt.Sprintf("%v", clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.Reloader)
+
 	clusterSummary := clusterSummaryScope.ClusterSummary
 	if clusterSummary.Spec.ClusterProfileSpec.HelmCharts == nil {
 		return h.Sum(nil), nil
@@ -1393,13 +1433,16 @@ func getInstantiatedValues(ctx context.Context, clusterSummary *configv1alpha1.C
 	return chartutil.ReadValues([]byte(instantiatedValues))
 }
 
-func deployResourceSummaryWithHelmResources(ctx context.Context, c client.Client,
-	clusterNamespace, clusterName string, clusterType libsveltosv1alpha1.ClusterType,
-	clusterSummary *configv1alpha1.ClusterSummary, kubeconfig string, logger logr.Logger) error {
+// collectResourcesFromManagedHelmCharts collects resources considering all
+// helm charts contained in a ClusterSummary that are currently managed by the
+// ClusterProfile instance
+func collectResourcesFromManagedHelmCharts(ctx context.Context, c client.Client,
+	clusterSummary *configv1alpha1.ClusterSummary, kubeconfig string, logger logr.Logger,
+) ([]libsveltosv1alpha1.HelmResources, error) {
 
 	chartManager, err := chartmanager.GetChartManagerInstance(ctx, c)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	helmResources := make([]libsveltosv1alpha1.HelmResources, 0)
@@ -1412,18 +1455,18 @@ func deployResourceSummaryWithHelmResources(ctx context.Context, c client.Client
 			actionConfig, err := actionConfigInit(currentChart.ReleaseNamespace, kubeconfig, logger)
 
 			if err != nil {
-				return err
+				return nil, err
 			}
 
 			statusObject := action.NewStatus(actionConfig)
 			results, err := statusObject.Run(currentChart.ReleaseName)
 			if err != nil {
-				return err
+				return nil, err
 			}
 
 			resources, err := collectHelmContent(results.Manifest, logger)
 			if err != nil {
-				return err
+				return nil, err
 			}
 
 			l.V(logs.LogDebug).Info(fmt.Sprintf("found %d resources", len(resources)))
@@ -1439,8 +1482,7 @@ func deployResourceSummaryWithHelmResources(ctx context.Context, c client.Client
 		}
 	}
 
-	return deployResourceSummaryInCluster(ctx, c, clusterNamespace, clusterName, clusterSummary.Name,
-		clusterType, nil, nil, helmResources, logger)
+	return helmResources, nil
 }
 
 func collectHelmContent(manifest string, logger logr.Logger) ([]*unstructured.Unstructured, error) {

controllers/handlers_helm_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"crypto/sha256"
 	"errors"
+	"fmt"
 	"reflect"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -607,7 +608,8 @@ var _ = Describe("Hash methods", func() {
 		})
 		Expect(err).To(BeNil())
 
-		config := render.AsCode(kyvernoChart)
+		config := fmt.Sprintf("%v", clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.Reloader)
+		config += render.AsCode(kyvernoChart)
 		config += render.AsCode(nginxChart)
 		h := sha256.New()
 		h.Write([]byte(config))

controllers/handlers_kustomize.go

@@ -103,6 +103,12 @@ func deployKustomizeRefs(ctx context.Context, c client.Client,
 	if err != nil {
 		return err
 	}
+	remoteResources := convertResourceReportsToObjectReference(remoteResourceReports)
+	err = updateReloaderWithDeployedResources(ctx, c, clusterProfileOwnerRef, configv1alpha1.FeatureKustomize,
+		remoteResources, clusterSummary, logger)
+	if err != nil {
+		return err
+	}
 
 	// If we are here there are no conflicts (and error would have been returned by deployKustomizeRef)
 	remoteDeployed := make([]configv1alpha1.Resource, 0)
@@ -116,15 +122,9 @@ func deployKustomizeRefs(ctx context.Context, c client.Client,
 		return err
 	}
 
-	// Clean stale resources in the management cluster
-	_, err = cleanKustomizeResources(ctx, getManagementClusterConfig(), c, clusterSummary, localResourceReports, logger)
-	if err != nil {
-		return err
-	}
-
-	// Clean stale resources in the remote cluster
 	var undeployed []configv1alpha1.ResourceReport
-	undeployed, err = cleanKustomizeResources(ctx, remoteRestConfig, remoteClient, clusterSummary, remoteResourceReports, logger)
+	_, undeployed, err = cleanStaleKustomizeResources(ctx, remoteRestConfig, remoteClient, clusterSummary,
+		localResourceReports, remoteResourceReports, logger)
 	if err != nil {
 		return err
 	}
@@ -152,6 +152,26 @@ func deployKustomizeRefs(ctx context.Context, c client.Client,
 	return nil
 }
 
+func cleanStaleKustomizeResources(ctx context.Context, remoteRestConfig *rest.Config, remoteClient client.Client,
+	clusterSummary *configv1alpha1.ClusterSummary, localResourceReports, remoteResourceReports []configv1alpha1.ResourceReport,
+	logger logr.Logger) (localUndeployed, remoteUndeployed []configv1alpha1.ResourceReport, err error) {
+	// Clean stale resources in the management cluster
+	localUndeployed, err = cleanKustomizeResources(ctx, getManagementClusterConfig(), getManagementClusterClient(),
+		clusterSummary, localResourceReports, logger)
+	if err != nil {
+		return
+	}
+
+	// Clean stale resources in the remote cluster
+	remoteUndeployed, err = cleanKustomizeResources(ctx, remoteRestConfig, remoteClient,
+		clusterSummary, remoteResourceReports, logger)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
 func undeployKustomizeRefs(ctx context.Context, c client.Client,
 	clusterNamespace, clusterName, applicant, _ string,
 	clusterType libsveltosv1alpha1.ClusterType,
@@ -207,6 +227,12 @@ func undeployKustomizeRefs(ctx context.Context, c client.Client,
 		return err
 	}
 
+	err = updateReloaderWithDeployedResources(ctx, c, clusterProfileOwnerRef, configv1alpha1.FeatureKustomize,
+		nil, clusterSummary, logger)
+	if err != nil {
+		return err
+	}
+
 	err = updateClusterConfiguration(ctx, c, clusterSummary, clusterProfileOwnerRef,
 		configv1alpha1.FeatureKustomize, []configv1alpha1.Resource{}, nil)
 	if err != nil {
@@ -235,6 +261,10 @@ func kustomizationHash(ctx context.Context, c client.Client, clusterSummaryScope
 	h := sha256.New()
 	var config string
 
+	// If Reloader changes, Reloader needs to be deployed or undeployed
+	// So consider it in the hash
+	config += fmt.Sprintf("%v", clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.Reloader)
+
 	config += render.AsCode(clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.KustomizationRefs)
 
 	for i := range clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.KustomizationRefs {

controllers/handlers_kustomize_test.go

@@ -21,6 +21,7 @@ import (
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -295,7 +296,7 @@ var _ = Describe("Hash methods", func() {
 		})
 		Expect(err).To(BeNil())
 
-		var config string
+		config := fmt.Sprintf("%v", clusterSummaryScope.ClusterSummary.Spec.ClusterProfileSpec.Reloader)
 		config += render.AsCode(clusterSummary.Spec.ClusterProfileSpec.KustomizationRefs)
 		for i := 0; i < repoNum; i++ {
 			config += gitRepositories[i].Status.Artifact.Revision