From 03f15953fbdf0786d55873d690cd1e18267e58cb Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Mon, 24 Sep 2018 16:20:22 -0700 Subject: [PATCH] Actually allow cloudprovider rate limit / backoff disabling (#3891) --- pkg/acsengine/const.go | 16 +-- pkg/acsengine/defaults.go | 35 ++---- pkg/acsengine/defaults_test.go | 34 ++++++ pkg/acsengine/k8s_versions.go | 85 ++++++++------- pkg/api/const.go | 15 +++ pkg/api/types.go | 34 +++++- pkg/api/types_test.go | 194 +++++++++++++++++++++++++++++++++ pkg/api/vlabs/types.go | 4 +- pkg/api/vlabs/validate_test.go | 8 +- 9 files changed, 335 insertions(+), 90 deletions(-) diff --git a/pkg/acsengine/const.go b/pkg/acsengine/const.go index 72e134b781..b8da1a6978 100644 --- a/pkg/acsengine/const.go +++ b/pkg/acsengine/const.go @@ -83,21 +83,9 @@ const ( // DefaultKubernetesCtrlMgrUseSvcAccountCreds is "true", see --use-service-account-credentials at https://kubernetes.io/docs/admin/kube-controller-manager/ DefaultKubernetesCtrlMgrUseSvcAccountCreds = "false" // DefaultKubernetesCloudProviderBackoff is false to disable cloudprovider backoff implementation for API calls - DefaultKubernetesCloudProviderBackoff = false - // DefaultKubernetesCloudProviderBackoffRetries is 6, takes effect if DefaultKubernetesCloudProviderBackoff is true - DefaultKubernetesCloudProviderBackoffRetries = 6 - // DefaultKubernetesCloudProviderBackoffJitter is 1, takes effect if DefaultKubernetesCloudProviderBackoff is true - DefaultKubernetesCloudProviderBackoffJitter = 1.0 - // DefaultKubernetesCloudProviderBackoffDuration is 5, takes effect if DefaultKubernetesCloudProviderBackoff is true - DefaultKubernetesCloudProviderBackoffDuration = 5 - // DefaultKubernetesCloudProviderBackoffExponent is 1.5, takes effect if DefaultKubernetesCloudProviderBackoff is true - DefaultKubernetesCloudProviderBackoffExponent = 1.5 + DefaultKubernetesCloudProviderBackoff = true // DefaultKubernetesCloudProviderRateLimit is false to disable cloudprovider rate limiting implementation for API calls - DefaultKubernetesCloudProviderRateLimit = false - // DefaultKubernetesCloudProviderRateLimitQPS is 3, takes effect if DefaultKubernetesCloudProviderRateLimit is true - DefaultKubernetesCloudProviderRateLimitQPS = 3.0 - // DefaultKubernetesCloudProviderRateLimitBucket is 10, takes effect if DefaultKubernetesCloudProviderRateLimit is true - DefaultKubernetesCloudProviderRateLimitBucket = 10 + DefaultKubernetesCloudProviderRateLimit = true // DefaultKubeHeapsterDeploymentAddonName is the name of the kube-heapster-deployment addon DefaultKubeHeapsterDeploymentAddonName = "kube-heapster-deployment" // DefaultKubeDNSDeploymentAddonName is the name of the kube-dns-deployment addon diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 9a4645ddfc..852ab90b35 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -278,8 +278,6 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) { switch o.OrchestratorType { case api.Kubernetes: - k8sVersion := o.OrchestratorVersion - if o.KubernetesConfig == nil { o.KubernetesConfig = &api.KubernetesConfig{} } @@ -342,32 +340,21 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) { if o.KubernetesConfig.ServiceCIDR == "" { o.KubernetesConfig.ServiceCIDR = DefaultKubernetesServiceCIDR } - // Enforce sane cloudprovider backoff defaults, if CloudProviderBackoff is true in KubernetesConfig - o.KubernetesConfig.CloudProviderBackoff = true - if o.KubernetesConfig.CloudProviderBackoffDuration == 0 { - o.KubernetesConfig.CloudProviderBackoffDuration = DefaultKubernetesCloudProviderBackoffDuration - } - if o.KubernetesConfig.CloudProviderBackoffExponent == 0 { - o.KubernetesConfig.CloudProviderBackoffExponent = DefaultKubernetesCloudProviderBackoffExponent + + if o.KubernetesConfig.CloudProviderBackoff == nil { + o.KubernetesConfig.CloudProviderBackoff = helpers.PointerToBool(DefaultKubernetesCloudProviderBackoff) } - if o.KubernetesConfig.CloudProviderBackoffJitter == 0 { - o.KubernetesConfig.CloudProviderBackoffJitter = DefaultKubernetesCloudProviderBackoffJitter + // Enforce sane cloudprovider backoff defaults, if CloudProviderBackoff is true in KubernetesConfig + if helpers.IsTrueBoolPointer(o.KubernetesConfig.CloudProviderBackoff) { + o.KubernetesConfig.SetCloudProviderBackoffDefaults() } - if o.KubernetesConfig.CloudProviderBackoffRetries == 0 { - o.KubernetesConfig.CloudProviderBackoffRetries = DefaultKubernetesCloudProviderBackoffRetries + + if o.KubernetesConfig.CloudProviderRateLimit == nil { + o.KubernetesConfig.CloudProviderRateLimit = helpers.PointerToBool(DefaultKubernetesCloudProviderRateLimit) } - k8sSemVer, _ := semver.Make(k8sVersion) - minVersion, _ := semver.Make("1.6.6") // Enforce sane cloudprovider rate limit defaults, if CloudProviderRateLimit is true in KubernetesConfig - // For k8s version greater or equal to 1.6.6, we will set the default CloudProviderRate* settings - o.KubernetesConfig.CloudProviderRateLimit = true - if o.KubernetesConfig.CloudProviderRateLimit && k8sSemVer.GTE(minVersion) { - if o.KubernetesConfig.CloudProviderRateLimitQPS == 0 { - o.KubernetesConfig.CloudProviderRateLimitQPS = DefaultKubernetesCloudProviderRateLimitQPS - } - if o.KubernetesConfig.CloudProviderRateLimitBucket == 0 { - o.KubernetesConfig.CloudProviderRateLimitBucket = DefaultKubernetesCloudProviderRateLimitBucket - } + if helpers.IsTrueBoolPointer(o.KubernetesConfig.CloudProviderRateLimit) { + o.KubernetesConfig.SetCloudProviderRateLimitDefaults() } if o.KubernetesConfig.PrivateCluster == nil { diff --git a/pkg/acsengine/defaults_test.go b/pkg/acsengine/defaults_test.go index aa236f1127..c53644a017 100644 --- a/pkg/acsengine/defaults_test.go +++ b/pkg/acsengine/defaults_test.go @@ -679,6 +679,40 @@ func TestDefaultDisableRbac(t *testing.T) { } } +func TestDefaultCloudProvider(t *testing.T) { + mockCS := getMockBaseContainerService("1.10.3") + properties := mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + setOrchestratorDefaults(&mockCS, true) + + if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff) { + t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t", + helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) + } + + if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit) { + t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t", + helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) + } + + mockCS = getMockBaseContainerService("1.10.3") + properties = mockCS.Properties + properties.OrchestratorProfile.OrchestratorType = "Kubernetes" + properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff = helpers.PointerToBool(false) + properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit = helpers.PointerToBool(false) + setOrchestratorDefaults(&mockCS, true) + + if !helpers.IsFalseBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff) { + t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t", + helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) + } + + if !helpers.IsFalseBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit) { + t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t", + helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff)) + } +} + func getMockAddon(name string) api.KubernetesAddon { return api.KubernetesAddon{ Name: name, diff --git a/pkg/acsengine/k8s_versions.go b/pkg/acsengine/k8s_versions.go index 62e884996d..9b898a7422 100644 --- a/pkg/acsengine/k8s_versions.go +++ b/pkg/acsengine/k8s_versions.go @@ -4,6 +4,7 @@ import ( "strconv" "strings" + "github.com/Azure/acs-engine/pkg/api" "github.com/Azure/acs-engine/pkg/api/common" ) @@ -31,12 +32,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, @@ -63,12 +64,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, @@ -95,12 +96,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, @@ -126,12 +127,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, @@ -155,12 +156,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, @@ -184,12 +185,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, @@ -212,12 +213,12 @@ var k8sComponentVersions = map[string]map[string]string{ "nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod, "podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout, "routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod, - "backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries), - "backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), - "backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration), - "backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), - "ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), - "ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket), + "backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries), + "backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64), + "backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration), + "backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64), + "ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64), + "ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket), "gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold), "gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold), }, diff --git a/pkg/api/const.go b/pkg/api/const.go index c3d9ad0c8b..8f2a618340 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -177,3 +177,18 @@ const ( // VHDDiskSizeAKS maps to the OSDiskSizeGB for AKS VHD image VHDDiskSizeAKS = 30 ) + +const ( + // DefaultKubernetesCloudProviderBackoffRetries is 6, takes effect if DefaultKubernetesCloudProviderBackoff is true + DefaultKubernetesCloudProviderBackoffRetries = 6 + // DefaultKubernetesCloudProviderBackoffJitter is 1, takes effect if DefaultKubernetesCloudProviderBackoff is true + DefaultKubernetesCloudProviderBackoffJitter = 1.0 + // DefaultKubernetesCloudProviderBackoffDuration is 5, takes effect if DefaultKubernetesCloudProviderBackoff is true + DefaultKubernetesCloudProviderBackoffDuration = 5 + // DefaultKubernetesCloudProviderBackoffExponent is 1.5, takes effect if DefaultKubernetesCloudProviderBackoff is true + DefaultKubernetesCloudProviderBackoffExponent = 1.5 + // DefaultKubernetesCloudProviderRateLimitQPS is 3, takes effect if DefaultKubernetesCloudProviderRateLimit is true + DefaultKubernetesCloudProviderRateLimitQPS = 3.0 + // DefaultKubernetesCloudProviderRateLimitBucket is 10, takes effect if DefaultKubernetesCloudProviderRateLimit is true + DefaultKubernetesCloudProviderRateLimitBucket = 10 +) diff --git a/pkg/api/types.go b/pkg/api/types.go index 6efbd60966..51eeb3049e 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -254,12 +254,12 @@ type PrivateJumpboxProfile struct { // CloudProviderConfig contains the KubernetesConfig properties specific to the Cloud Provider type CloudProviderConfig struct { - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` CloudProviderBackoffJitter string `json:"cloudProviderBackoffJitter,omitempty"` CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` CloudProviderBackoffExponent string `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` CloudProviderRateLimitQPS string `json:"cloudProviderRateLimitQPS,omitempty"` CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` } @@ -314,12 +314,12 @@ type KubernetesConfig struct { CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` NonMasqueradeCidr string `json:"nonMasqueradeCidr,omitempty"` @@ -1009,3 +1009,29 @@ func (k *KubernetesConfig) RequiresDocker() bool { runtime := strings.ToLower(k.ContainerRuntime) return runtime == "docker" || runtime == "" } + +// SetCloudProviderBackoffDefaults sets default cloudprovider backoff config +func (k *KubernetesConfig) SetCloudProviderBackoffDefaults() { + if k.CloudProviderBackoffDuration == 0 { + k.CloudProviderBackoffDuration = DefaultKubernetesCloudProviderBackoffDuration + } + if k.CloudProviderBackoffExponent == 0 { + k.CloudProviderBackoffExponent = DefaultKubernetesCloudProviderBackoffExponent + } + if k.CloudProviderBackoffJitter == 0 { + k.CloudProviderBackoffJitter = DefaultKubernetesCloudProviderBackoffJitter + } + if k.CloudProviderBackoffRetries == 0 { + k.CloudProviderBackoffRetries = DefaultKubernetesCloudProviderBackoffRetries + } +} + +// SetCloudProviderRateLimitDefaults sets default cloudprovider rate limiter config +func (k *KubernetesConfig) SetCloudProviderRateLimitDefaults() { + if k.CloudProviderRateLimitQPS == 0 { + k.CloudProviderRateLimitQPS = DefaultKubernetesCloudProviderRateLimitQPS + } + if k.CloudProviderRateLimitBucket == 0 { + k.CloudProviderRateLimitBucket = DefaultKubernetesCloudProviderRateLimitBucket + } +} diff --git a/pkg/api/types_test.go b/pkg/api/types_test.go index 12723af7af..470f19dda7 100644 --- a/pkg/api/types_test.go +++ b/pkg/api/types_test.go @@ -1257,6 +1257,200 @@ func TestIsMetricsServerEnabled(t *testing.T) { } } +func TestCloudProviderDefaults(t *testing.T) { + // Test cloudprovider defaults when no user-provided values + v := "1.8.0" + o := OrchestratorProfile{ + OrchestratorType: "Kubernetes", + OrchestratorVersion: v, + KubernetesConfig: &KubernetesConfig{}, + } + o.KubernetesConfig.SetCloudProviderBackoffDefaults() + o.KubernetesConfig.SetCloudProviderRateLimitDefaults() + + intCases := []struct { + defaultVal int + computedVal int + }{ + { + defaultVal: DefaultKubernetesCloudProviderBackoffRetries, + computedVal: o.KubernetesConfig.CloudProviderBackoffRetries, + }, + { + defaultVal: DefaultKubernetesCloudProviderBackoffDuration, + computedVal: o.KubernetesConfig.CloudProviderBackoffDuration, + }, + { + defaultVal: DefaultKubernetesCloudProviderRateLimitBucket, + computedVal: o.KubernetesConfig.CloudProviderRateLimitBucket, + }, + } + + for _, c := range intCases { + if c.computedVal != c.defaultVal { + t.Fatalf("KubernetesConfig empty cloudprovider configs should reflect default values after SetCloudProviderBackoffDefaults(), expected %d, got %d", c.defaultVal, c.computedVal) + } + } + + floatCases := []struct { + defaultVal float64 + computedVal float64 + }{ + { + defaultVal: DefaultKubernetesCloudProviderBackoffJitter, + computedVal: o.KubernetesConfig.CloudProviderBackoffJitter, + }, + { + defaultVal: DefaultKubernetesCloudProviderBackoffExponent, + computedVal: o.KubernetesConfig.CloudProviderBackoffExponent, + }, + { + defaultVal: DefaultKubernetesCloudProviderRateLimitQPS, + computedVal: o.KubernetesConfig.CloudProviderRateLimitQPS, + }, + } + + for _, c := range floatCases { + if c.computedVal != c.defaultVal { + t.Fatalf("KubernetesConfig empty cloudprovider configs should reflect default values after SetCloudProviderBackoffDefaults(), expected %f, got %f", c.defaultVal, c.computedVal) + } + } + + customCloudProviderBackoffDuration := 99 + customCloudProviderBackoffExponent := 10.0 + customCloudProviderBackoffJitter := 11.9 + customCloudProviderBackoffRetries := 9 + customCloudProviderRateLimitBucket := 37 + customCloudProviderRateLimitQPS := 9.9 + + // Test cloudprovider defaults when user provides configuration + v = "1.8.0" + o = OrchestratorProfile{ + OrchestratorType: "Kubernetes", + OrchestratorVersion: v, + KubernetesConfig: &KubernetesConfig{ + CloudProviderBackoffDuration: customCloudProviderBackoffDuration, + CloudProviderBackoffExponent: customCloudProviderBackoffExponent, + CloudProviderBackoffJitter: customCloudProviderBackoffJitter, + CloudProviderBackoffRetries: customCloudProviderBackoffRetries, + CloudProviderRateLimitBucket: customCloudProviderRateLimitBucket, + CloudProviderRateLimitQPS: customCloudProviderRateLimitQPS, + }, + } + o.KubernetesConfig.SetCloudProviderBackoffDefaults() + o.KubernetesConfig.SetCloudProviderRateLimitDefaults() + + intCasesCustom := []struct { + customVal int + computedVal int + }{ + { + customVal: customCloudProviderBackoffRetries, + computedVal: o.KubernetesConfig.CloudProviderBackoffRetries, + }, + { + customVal: customCloudProviderBackoffDuration, + computedVal: o.KubernetesConfig.CloudProviderBackoffDuration, + }, + { + customVal: customCloudProviderRateLimitBucket, + computedVal: o.KubernetesConfig.CloudProviderRateLimitBucket, + }, + } + + for _, c := range intCasesCustom { + if c.computedVal != c.customVal { + t.Fatalf("KubernetesConfig empty cloudprovider configs should reflect default values after SetCloudProviderBackoffDefaults(), expected %d, got %d", c.customVal, c.computedVal) + } + } + + floatCasesCustom := []struct { + customVal float64 + computedVal float64 + }{ + { + customVal: customCloudProviderBackoffJitter, + computedVal: o.KubernetesConfig.CloudProviderBackoffJitter, + }, + { + customVal: customCloudProviderBackoffExponent, + computedVal: o.KubernetesConfig.CloudProviderBackoffExponent, + }, + { + customVal: customCloudProviderRateLimitQPS, + computedVal: o.KubernetesConfig.CloudProviderRateLimitQPS, + }, + } + + for _, c := range floatCasesCustom { + if c.computedVal != c.customVal { + t.Fatalf("KubernetesConfig empty cloudprovider configs should reflect default values after SetCloudProviderBackoffDefaults(), expected %f, got %f", c.customVal, c.computedVal) + } + } + + // Test cloudprovider defaults when user provides *some* config values + v = "1.8.0" + o = OrchestratorProfile{ + OrchestratorType: "Kubernetes", + OrchestratorVersion: v, + KubernetesConfig: &KubernetesConfig{ + CloudProviderBackoffDuration: customCloudProviderBackoffDuration, + CloudProviderRateLimitBucket: customCloudProviderRateLimitBucket, + CloudProviderRateLimitQPS: customCloudProviderRateLimitQPS, + }, + } + o.KubernetesConfig.SetCloudProviderBackoffDefaults() + o.KubernetesConfig.SetCloudProviderRateLimitDefaults() + + intCasesMixed := []struct { + expectedVal int + computedVal int + }{ + { + expectedVal: DefaultKubernetesCloudProviderBackoffRetries, + computedVal: o.KubernetesConfig.CloudProviderBackoffRetries, + }, + { + expectedVal: customCloudProviderBackoffDuration, + computedVal: o.KubernetesConfig.CloudProviderBackoffDuration, + }, + { + expectedVal: customCloudProviderRateLimitBucket, + computedVal: o.KubernetesConfig.CloudProviderRateLimitBucket, + }, + } + + for _, c := range intCasesMixed { + if c.computedVal != c.expectedVal { + t.Fatalf("KubernetesConfig empty cloudprovider configs should reflect default values after SetCloudProviderBackoffDefaults(), expected %d, got %d", c.expectedVal, c.computedVal) + } + } + + floatCasesMixed := []struct { + expectedVal float64 + computedVal float64 + }{ + { + expectedVal: DefaultKubernetesCloudProviderBackoffJitter, + computedVal: o.KubernetesConfig.CloudProviderBackoffJitter, + }, + { + expectedVal: DefaultKubernetesCloudProviderBackoffExponent, + computedVal: o.KubernetesConfig.CloudProviderBackoffExponent, + }, + { + expectedVal: customCloudProviderRateLimitQPS, + computedVal: o.KubernetesConfig.CloudProviderRateLimitQPS, + }, + } + + for _, c := range floatCasesMixed { + if c.computedVal != c.expectedVal { + t.Fatalf("KubernetesConfig empty cloudprovider configs should reflect default values after SetCloudProviderBackoffDefaults(), expected %f, got %f", c.expectedVal, c.computedVal) + } + } +} + func getMockAddon(name string) KubernetesAddon { return KubernetesAddon{ Name: name, diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 12ec7ee82b..869f77fa41 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -300,12 +300,12 @@ type KubernetesConfig struct { CloudControllerManagerConfig map[string]string `json:"cloudControllerManagerConfig,omitempty"` APIServerConfig map[string]string `json:"apiServerConfig,omitempty"` SchedulerConfig map[string]string `json:"schedulerConfig,omitempty"` - CloudProviderBackoff bool `json:"cloudProviderBackoff,omitempty"` + CloudProviderBackoff *bool `json:"cloudProviderBackoff,omitempty"` CloudProviderBackoffRetries int `json:"cloudProviderBackoffRetries,omitempty"` CloudProviderBackoffJitter float64 `json:"cloudProviderBackoffJitter,omitempty"` CloudProviderBackoffDuration int `json:"cloudProviderBackoffDuration,omitempty"` CloudProviderBackoffExponent float64 `json:"cloudProviderBackoffExponent,omitempty"` - CloudProviderRateLimit bool `json:"cloudProviderRateLimit,omitempty"` + CloudProviderRateLimit *bool `json:"cloudProviderRateLimit,omitempty"` CloudProviderRateLimitQPS float64 `json:"cloudProviderRateLimitQPS,omitempty"` CloudProviderRateLimitBucket int `json:"cloudProviderRateLimitBucket,omitempty"` LoadBalancerSku string `json:"loadBalancerSku,omitempty"` diff --git a/pkg/api/vlabs/validate_test.go b/pkg/api/vlabs/validate_test.go index 362267878e..87f10e2f8e 100644 --- a/pkg/api/vlabs/validate_test.go +++ b/pkg/api/vlabs/validate_test.go @@ -368,12 +368,12 @@ func Test_KubernetesConfig_Validate(t *testing.T) { ClusterSubnet: "10.120.0.0/16", DockerBridgeSubnet: "10.120.1.0/16", MaxPods: 42, - CloudProviderBackoff: ValidKubernetesCloudProviderBackoff, + CloudProviderBackoff: helpers.PointerToBool(ValidKubernetesCloudProviderBackoff), CloudProviderBackoffRetries: ValidKubernetesCloudProviderBackoffRetries, CloudProviderBackoffJitter: ValidKubernetesCloudProviderBackoffJitter, CloudProviderBackoffDuration: ValidKubernetesCloudProviderBackoffDuration, CloudProviderBackoffExponent: ValidKubernetesCloudProviderBackoffExponent, - CloudProviderRateLimit: ValidKubernetesCloudProviderRateLimit, + CloudProviderRateLimit: helpers.PointerToBool(ValidKubernetesCloudProviderRateLimit), CloudProviderRateLimitQPS: ValidKubernetesCloudProviderRateLimitQPS, CloudProviderRateLimitBucket: ValidKubernetesCloudProviderRateLimitBucket, KubeletConfig: map[string]string{ @@ -551,8 +551,8 @@ func Test_KubernetesConfig_Validate(t *testing.T) { // Tests that apply to 1.6 and later releases for _, k8sVersion := range common.GetAllSupportedKubernetesVersions(false, false) { c := KubernetesConfig{ - CloudProviderBackoff: true, - CloudProviderRateLimit: true, + CloudProviderBackoff: helpers.PointerToBool(true), + CloudProviderRateLimit: helpers.PointerToBool(true), } if err := c.Validate(k8sVersion, false); err != nil { t.Error("should not error when basic backoff and rate limiting are set to true with no options")