Sprig Templating for Helm Values with Inputs from Cluster Resource

.gitignore

@@ -11,3 +11,4 @@
 docs/fleet-agent/
 docs/fleet-cli/
 docs/fleet-controller/
+^fleet$

charts/fleet-crd/templates/crds.yaml

@@ -117,6 +117,8 @@ spec:
                   chart:
                     nullable: true
                     type: string
+                  disablePreProcess:
+                    type: boolean
                   force:
                     type: boolean
                   maxHistory:
@@ -488,6 +490,8 @@ spec:
                         chart:
                           nullable: true
                           type: string
+                        disablePreProcess:
+                          type: boolean
                         force:
                           type: boolean
                         maxHistory:
@@ -997,6 +1001,8 @@ spec:
                       chart:
                         nullable: true
                         type: string
+                      disablePreProcess:
+                        type: boolean
                       force:
                         type: boolean
                       maxHistory:
@@ -1144,6 +1150,8 @@ spec:
                       chart:
                         nullable: true
                         type: string
+                      disablePreProcess:
+                        type: boolean
                       force:
                         type: boolean
                       maxHistory:
@@ -1784,6 +1792,10 @@ spec:
                 type: string
               redeployAgentGeneration:
                 type: integer
+              templateValues:
+                nullable: true
+                type: object
+                x-kubernetes-preserve-unknown-fields: true
             type: object
           status:
             properties:

go.mod

@@ -33,7 +33,8 @@ replace (
 )
 
 require (
-	github.com/Masterminds/semver/v3 v3.2.0
+	github.com/Masterminds/semver/v3 v3.1.1
+	github.com/Masterminds/sprig/v3 v3.2.2
 	github.com/cheggaaa/pb v1.0.29
 	github.com/davecgh/go-spew v1.1.1
 	github.com/evanphx/json-patch v5.6.0+incompatible
@@ -93,7 +94,6 @@ require (
 	github.com/BurntSushi/toml v1.1.0 // indirect
 	github.com/MakeNowJust/heredoc v1.0.0 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
-	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
 	github.com/Masterminds/squirrel v1.5.3 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20220623141421-5afb4c282135 // indirect

go.sum

@@ -115,9 +115,8 @@ github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy86
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
-github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
-github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
 github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8=
 github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk=

pkg/apis/fleet.cattle.io/v1alpha1/bundle.go

@@ -237,6 +237,9 @@ type HelmOptions struct {
 
 	// Atomic sets the --atomic flag when Helm is performing an upgrade
 	Atomic bool `json:"atomic,omitempty"`
+
+	// DisablePreProcess disables template processing in values
+	DisablePreProcess bool `json:"disablePreProcess,omitempty"`
 }
 
 // Define helm values that can come from configmap, secret or external. Credit: https://github.com/fluxcd/helm-operator/blob/0cfea875b5d44bea995abe7324819432070dfbdc/pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go#L439

pkg/apis/fleet.cattle.io/v1alpha1/target.go

@@ -70,6 +70,9 @@ type ClusterSpec struct {
 	// AgentNamespace defaults to the system namespace, e.g. cattle-fleet-system
 	AgentNamespace string `json:"agentNamespace,omitempty"`
 	PrivateRepoURL string `json:"privateRepoURL,omitempty"`
+
+	// TemplateValues defines a cluster specific mapping of values to be sent to fleet.yaml values templating
+	TemplateValues *GenericMap `json:"templateValues,omitempty"`
 }
 
 type ClusterStatus struct {

pkg/options/calculate.go

@@ -81,6 +81,7 @@ func Merge(base, next fleet.BundleDeploymentOptions) fleet.BundleDeploymentOptio
 		result.Helm.Force = result.Helm.Force || next.Helm.Force
 		result.Helm.Atomic = result.Helm.Atomic || next.Helm.Atomic
 		result.Helm.TakeOwnership = result.Helm.TakeOwnership || next.Helm.TakeOwnership
+		result.Helm.DisablePreProcess = result.Helm.DisablePreProcess || next.Helm.DisablePreProcess
 	}
 	if next.Kustomize != nil {
 		if result.Kustomize == nil {

pkg/target/target.go

@@ -6,10 +6,12 @@
 package target
 
 import (
+	"bytes"
 	"fmt"
 	"sort"
 	"strconv"
 	"strings"
+	"text/template"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -21,7 +23,6 @@ import (
 	"github.com/rancher/fleet/pkg/options"
 	"github.com/rancher/fleet/pkg/summary"
 
-	"github.com/rancher/wrangler/pkg/data"
 	corecontrollers "github.com/rancher/wrangler/pkg/generated/controllers/core/v1"
 	"github.com/rancher/wrangler/pkg/name"
 	"github.com/rancher/wrangler/pkg/yaml"
@@ -30,6 +31,8 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
+
+	"github.com/Masterminds/sprig/v3"
 )
 
 var (
@@ -39,6 +42,8 @@ var (
 	defMaxUnavailablePartitions = intstr.FromInt(0)
 )
 
+const maxTemplateRecursionDepth = 50
+
 type Manager struct {
 	clusters                    fleetcontrollers.ClusterCache
 	clusterGroups               fleetcontrollers.ClusterGroupCache
@@ -263,7 +268,7 @@ func (m *Manager) Targets(bundle *fleet.Bundle, manifest *manifest.Manifest) ([]
 			}
 
 			opts := options.Merge(bundle.Spec.BundleDeploymentOptions, target.BundleDeploymentOptions)
-			err = addClusterLabels(&opts, cluster.Labels)
+			err = preprocessHelmValues(&opts, cluster)
 			if err != nil {
 				return nil, err
 			}
@@ -290,9 +295,11 @@ func (m *Manager) Targets(bundle *fleet.Bundle, manifest *manifest.Manifest) ([]
 	return targets, m.foldInDeployments(bundle, targets)
 }
 
-func addClusterLabels(opts *fleet.BundleDeploymentOptions, labels map[string]string) (err error) {
-	clusterLabels := yaml.CleanAnnotationsForExport(labels)
-	for k, v := range labels {
+func preprocessHelmValues(opts *fleet.BundleDeploymentOptions, cluster *fleet.Cluster) (err error) {
+	clusterLabels := yaml.CleanAnnotationsForExport(cluster.Labels)
+	clusterAnnotations := yaml.CleanAnnotationsForExport(cluster.Annotations)
+
+	for k, v := range cluster.Labels {
 		if strings.HasPrefix(k, "fleet.cattle.io/") || strings.HasPrefix(k, "management.cattle.io/") {
 			clusterLabels[k] = v
 		}
@@ -301,27 +308,15 @@ func addClusterLabels(opts *fleet.BundleDeploymentOptions, labels map[string]str
 		return
 	}
 
-	newValues := map[string]interface{}{
-		"global": map[string]interface{}{
-			"fleet": map[string]interface{}{
-				"clusterLabels": clusterLabels,
-			},
-		},
-	}
-
 	if opts.Helm == nil {
-		opts.Helm = &fleet.HelmOptions{
-			Values: &fleet.GenericMap{
-				Data: newValues,
-			},
-		}
+		opts.Helm = &fleet.HelmOptions{}
 		return nil
 	}
 
 	opts.Helm = opts.Helm.DeepCopy()
 	if opts.Helm.Values == nil || opts.Helm.Values.Data == nil {
 		opts.Helm.Values = &fleet.GenericMap{
-			Data: newValues,
+			Data: map[string]interface{}{},
 		}
 		return nil
 	}
@@ -330,7 +325,28 @@ func addClusterLabels(opts *fleet.BundleDeploymentOptions, labels map[string]str
 		return err
 	}
 
-	opts.Helm.Values.Data = data.MergeMaps(opts.Helm.Values.Data, newValues)
+	if !opts.Helm.DisablePreProcess {
+
+		templateValues := map[string]interface{}{}
+		if cluster.Spec.TemplateValues != nil {
+			templateValues = cluster.Spec.TemplateValues.Data
+		}
+
+		values := map[string]interface{}{
+			"ClusterNamespace":   cluster.Namespace,
+			"ClusterName":        cluster.Name,
+			"ClusterLabels":      clusterLabels,
+			"ClusterAnnotations": clusterAnnotations,
+			"ClusterValues":      templateValues,
+		}
+
+		opts.Helm.Values.Data, err = processTemplateValues(opts.Helm.Values.Data, values)
+		if err != nil {
+			return err
+		}
+		logrus.Debugf("preProcess completed for %v", opts.Helm.ReleaseName)
+	}
+
 	return nil
 
 }
@@ -565,6 +581,86 @@ func Summary(targets []*Target) fleet.BundleSummary {
 	return bundleSummary
 }
 
+// tplFuncMap returns a mapping of all of the functions from sprig but removes potentially dangerous operations
+func tplFuncMap() template.FuncMap {
+	f := sprig.TxtFuncMap()
+	delete(f, "env")
+	delete(f, "expandenv")
+	delete(f, "include")
+	delete(f, "tpl")
+
+	return f
+}
+
+func processTemplateValues(valuesMap map[string]interface{}, templateContext map[string]interface{}) (map[string]interface{}, error) {
+	tplFn := template.New("values").Funcs(tplFuncMap()).Option("missingkey=error")
+	recursionDepth := 0
+	tplResult, err := templateSubstitutions(valuesMap, templateContext, tplFn, recursionDepth)
+	if err != nil {
+		return nil, err
+	}
+	compiledYaml, ok := tplResult.(map[string]interface{})
+	if !ok {
+		return nil, fmt.Errorf("templated result was expected to be map[string]interface{}, got %T", tplResult)
+	}
+
+	return compiledYaml, nil
+}
+
+func templateSubstitutions(src interface{}, templateContext map[string]interface{}, tplFn *template.Template, recursionDepth int) (interface{}, error) {
+	if recursionDepth > maxTemplateRecursionDepth {
+		return nil, fmt.Errorf("maximum recursion depth of %v exceeded for current templating operation, too many nested values", maxTemplateRecursionDepth)
+	}
+
+	switch tplVal := src.(type) {
+	case string:
+		tpl, err := tplFn.Parse(tplVal)
+		if err != nil {
+			return nil, err
+		}
+
+		var tplBytes bytes.Buffer
+		defer func() {
+			if r := recover(); r != nil {
+				err = fmt.Errorf("failed to process template substitution for string '%s': [%v]", tplVal, err)
+			}
+		}()
+		err = tpl.Execute(&tplBytes, templateContext)
+		if err != nil {
+			return nil, fmt.Errorf("failed to process template substitution for string '%s': [%v]", tplVal, err)
+		}
+		return tplBytes.String(), nil
+	case map[string]interface{}:
+		newMap := make(map[string]interface{})
+		for key, val := range tplVal {
+			processedKey, err := templateSubstitutions(key, templateContext, tplFn, recursionDepth+1)
+			if err != nil {
+				return nil, err
+			}
+			keyAsString, ok := processedKey.(string)
+			if !ok {
+				return nil, fmt.Errorf("expected a string to be returned, but instead got [%T]", processedKey)
+			}
+			if newMap[keyAsString], err = templateSubstitutions(val, templateContext, tplFn, recursionDepth+1); err != nil {
+				return nil, err
+			}
+		}
+		return newMap, nil
+	case []interface{}:
+		newSlice := make([]interface{}, len(tplVal))
+		for i, v := range tplVal {
+			newVal, err := templateSubstitutions(v, templateContext, tplFn, recursionDepth+1)
+			if err != nil {
+				return nil, err
+			}
+			newSlice[i] = newVal
+		}
+		return newSlice, nil
+	default:
+		return tplVal, nil
+	}
+}
+
 func processLabelValues(valuesMap map[string]interface{}, clusterLabels map[string]string) error {
 	prefix := "global.fleet.clusterLabels."
 	for key, val := range valuesMap {