Actually allow cloudprovider rate limit / backoff disabling (#3891)

Azure · Sep 24, 2018 · 03f1595 · 03f1595
1 parent 781379b
commit 03f1595
Show file tree

Hide file tree

Showing 9 changed files with 335 additions and 90 deletions.
diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go
@@ -83,21 +83,9 @@ const (
 	// DefaultKubernetesCtrlMgrUseSvcAccountCreds is "true", see --use-service-account-credentials at https://kubernetes.io/docs/admin/kube-controller-manager/
 	DefaultKubernetesCtrlMgrUseSvcAccountCreds = "false"
 	// DefaultKubernetesCloudProviderBackoff is false to disable cloudprovider backoff implementation for API calls
-	DefaultKubernetesCloudProviderBackoff = false
-	// DefaultKubernetesCloudProviderBackoffRetries is 6, takes effect if DefaultKubernetesCloudProviderBackoff is true
-	DefaultKubernetesCloudProviderBackoffRetries = 6
-	// DefaultKubernetesCloudProviderBackoffJitter is 1, takes effect if DefaultKubernetesCloudProviderBackoff is true
-	DefaultKubernetesCloudProviderBackoffJitter = 1.0
-	// DefaultKubernetesCloudProviderBackoffDuration is 5, takes effect if DefaultKubernetesCloudProviderBackoff is true
-	DefaultKubernetesCloudProviderBackoffDuration = 5
-	// DefaultKubernetesCloudProviderBackoffExponent is 1.5, takes effect if DefaultKubernetesCloudProviderBackoff is true
-	DefaultKubernetesCloudProviderBackoffExponent = 1.5
+	DefaultKubernetesCloudProviderBackoff = true
 	// DefaultKubernetesCloudProviderRateLimit is false to disable cloudprovider rate limiting implementation for API calls
-	DefaultKubernetesCloudProviderRateLimit = false
-	// DefaultKubernetesCloudProviderRateLimitQPS is 3, takes effect if DefaultKubernetesCloudProviderRateLimit is true
-	DefaultKubernetesCloudProviderRateLimitQPS = 3.0
-	// DefaultKubernetesCloudProviderRateLimitBucket is 10, takes effect if DefaultKubernetesCloudProviderRateLimit is true
-	DefaultKubernetesCloudProviderRateLimitBucket = 10
+	DefaultKubernetesCloudProviderRateLimit = true
 	// DefaultKubeHeapsterDeploymentAddonName is the name of the kube-heapster-deployment addon
 	DefaultKubeHeapsterDeploymentAddonName = "kube-heapster-deployment"
 	// DefaultKubeDNSDeploymentAddonName is the name of the kube-dns-deployment addon

diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go
@@ -278,8 +278,6 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) {
 
 	switch o.OrchestratorType {
 	case api.Kubernetes:
-		k8sVersion := o.OrchestratorVersion
-
 		if o.KubernetesConfig == nil {
 			o.KubernetesConfig = &api.KubernetesConfig{}
 		}
@@ -342,32 +340,21 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) {
 		if o.KubernetesConfig.ServiceCIDR == "" {
 			o.KubernetesConfig.ServiceCIDR = DefaultKubernetesServiceCIDR
 		}
-		// Enforce sane cloudprovider backoff defaults, if CloudProviderBackoff is true in KubernetesConfig
-		o.KubernetesConfig.CloudProviderBackoff = true
-		if o.KubernetesConfig.CloudProviderBackoffDuration == 0 {
-			o.KubernetesConfig.CloudProviderBackoffDuration = DefaultKubernetesCloudProviderBackoffDuration
-		}
-		if o.KubernetesConfig.CloudProviderBackoffExponent == 0 {
-			o.KubernetesConfig.CloudProviderBackoffExponent = DefaultKubernetesCloudProviderBackoffExponent
+
+		if o.KubernetesConfig.CloudProviderBackoff == nil {
+			o.KubernetesConfig.CloudProviderBackoff = helpers.PointerToBool(DefaultKubernetesCloudProviderBackoff)
 		}
-		if o.KubernetesConfig.CloudProviderBackoffJitter == 0 {
-			o.KubernetesConfig.CloudProviderBackoffJitter = DefaultKubernetesCloudProviderBackoffJitter
+		// Enforce sane cloudprovider backoff defaults, if CloudProviderBackoff is true in KubernetesConfig
+		if helpers.IsTrueBoolPointer(o.KubernetesConfig.CloudProviderBackoff) {
+			o.KubernetesConfig.SetCloudProviderBackoffDefaults()
 		}
-		if o.KubernetesConfig.CloudProviderBackoffRetries == 0 {
-			o.KubernetesConfig.CloudProviderBackoffRetries = DefaultKubernetesCloudProviderBackoffRetries
+
+		if o.KubernetesConfig.CloudProviderRateLimit == nil {
+			o.KubernetesConfig.CloudProviderRateLimit = helpers.PointerToBool(DefaultKubernetesCloudProviderRateLimit)
 		}
-		k8sSemVer, _ := semver.Make(k8sVersion)
-		minVersion, _ := semver.Make("1.6.6")
 		// Enforce sane cloudprovider rate limit defaults, if CloudProviderRateLimit is true in KubernetesConfig
-		// For k8s version greater or equal to 1.6.6, we will set the default CloudProviderRate* settings
-		o.KubernetesConfig.CloudProviderRateLimit = true
-		if o.KubernetesConfig.CloudProviderRateLimit && k8sSemVer.GTE(minVersion) {
-			if o.KubernetesConfig.CloudProviderRateLimitQPS == 0 {
-				o.KubernetesConfig.CloudProviderRateLimitQPS = DefaultKubernetesCloudProviderRateLimitQPS
-			}
-			if o.KubernetesConfig.CloudProviderRateLimitBucket == 0 {
-				o.KubernetesConfig.CloudProviderRateLimitBucket = DefaultKubernetesCloudProviderRateLimitBucket
-			}
+		if helpers.IsTrueBoolPointer(o.KubernetesConfig.CloudProviderRateLimit) {
+			o.KubernetesConfig.SetCloudProviderRateLimitDefaults()
 		}
 
 		if o.KubernetesConfig.PrivateCluster == nil {

diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go
@@ -679,6 +679,40 @@ func TestDefaultDisableRbac(t *testing.T) {
 	}
 }
 
+func TestDefaultCloudProvider(t *testing.T) {
+	mockCS := getMockBaseContainerService("1.10.3")
+	properties := mockCS.Properties
+	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
+	setOrchestratorDefaults(&mockCS, true)
+
+	if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff) {
+		t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
+			helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
+	}
+
+	if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit) {
+		t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
+			helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
+	}
+
+	mockCS = getMockBaseContainerService("1.10.3")
+	properties = mockCS.Properties
+	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
+	properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff = helpers.PointerToBool(false)
+	properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit = helpers.PointerToBool(false)
+	setOrchestratorDefaults(&mockCS, true)
+
+	if !helpers.IsFalseBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff) {
+		t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
+			helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
+	}
+
+	if !helpers.IsFalseBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit) {
+		t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
+			helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
+	}
+}
+
 func getMockAddon(name string) api.KubernetesAddon {
 	return api.KubernetesAddon{
 		Name:    name,

diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go
@@ -4,6 +4,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/Azure/acs-engine/pkg/api"
 	"github.com/Azure/acs-engine/pkg/api/common"
 )
 
@@ -31,12 +32,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
@@ -63,12 +64,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
@@ -95,12 +96,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
@@ -126,12 +127,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
@@ -155,12 +156,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
@@ -184,12 +185,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},
@@ -212,12 +213,12 @@ var k8sComponentVersions = map[string]map[string]string{
 		"nodegraceperiod":                  DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
 		"podeviction":                      DefaultKubernetesCtrlMgrPodEvictionTimeout,
 		"routeperiod":                      DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
-		"backoffretries":                   strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
-		"backoffjitter":                    strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
-		"backoffduration":                  strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
-		"backoffexponent":                  strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
-		"ratelimitqps":                     strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
-		"ratelimitbucket":                  strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
+		"backoffretries":                   strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
+		"backoffjitter":                    strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
+		"backoffduration":                  strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
+		"backoffexponent":                  strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
+		"ratelimitqps":                     strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
+		"ratelimitbucket":                  strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
 		"gchighthreshold":                  strconv.Itoa(DefaultKubernetesGCHighThreshold),
 		"gclowthreshold":                   strconv.Itoa(DefaultKubernetesGCLowThreshold),
 	},