From 6892be21a19170f9cd815f4492a1e35f6029ec0c Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 13:57:44 +0400 Subject: [PATCH 01/10] feat(DMVP-5664): flagger canary deployment ability in base helm chart --- charts/base/templates/rollout-strategy.yaml | 60 +++++++++++++++++ charts/base/templates/service.yaml | 2 +- charts/base/values.yaml | 67 ++++++++++++++++++- ...r-canary-nginx-custom.metric-template.yaml | 62 +++++++++++++++++ 4 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 charts/base/templates/rollout-strategy.yaml create mode 100644 examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml diff --git a/charts/base/templates/rollout-strategy.yaml b/charts/base/templates/rollout-strategy.yaml new file mode 100644 index 0000000..a40a4f2 --- /dev/null +++ b/charts/base/templates/rollout-strategy.yaml @@ -0,0 +1,60 @@ +{{- if .Values.rolloutStrategy.enabled -}} +{{- if eq .Values.rolloutStrategy.operator "flagger" -}} +apiVersion: flagger.app/v1beta1 +kind: Canary +metadata: + name: {{ include "base.fullname" . }} +spec: + provider: {{ .Values.rolloutStrategy.configs.provider | default "nginx" }} + # deployment reference + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "base.fullname" . }} + # ingress reference (TODO: there is supports for multiple ingresses, check and implement multiple ingress support here also ) + ingressRef: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: {{ include "base.fullname" . }} + {{- if .Values.autoscaling.enabled }} + # HPA reference (optional) + autoscalerRef: + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + name: {{ include "base.fullname" . }} + primaryScalerReplicas: + # min and max replicas count for primary hpa, default to main app hpa, the main app hpa values also being used for canary deploy hpa so we use this options to have custom values for primary hpa + minReplicas: {{ .Values.rolloutStrategy.configs.primaryScalerMinReplicas | default .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.rolloutStrategy.configs.primaryScalerMaxReplicas | default .Values.autoscaling.maxReplicas }} + {{- end }} + + # the maximum time in seconds for the canary deployment to make progress before it is rollback (default 600s) + progressDeadlineSeconds: {{ .Values.rolloutStrategy.configs.progressDeadlineSeconds | default 600 }} + service: + # ClusterIP port number + port: {{ .Values.service.port }} + # container port number or name + targetPort: {{ .Values.service.targetPort | default .Values.containerPort }} + analysis: + # minimum percentage of canary pods that must be ready before considering canary ready for traffic shifting (default 100) + canaryReadyThreshold: {{ .Values.rolloutStrategy.configs.canaryReadyThreshold | default 100 }} + # minimum percentage of primary pods that must be ready before considering primary ready for traffic shifting (default 100) + primaryReadyThreshold: {{ .Values.rolloutStrategy.configs.primaryReadyThreshold | default 100 }} + # schedule interval (default 60s) + interval: {{ .Values.rolloutStrategy.configs.interval | default "60s" }} + # max number of failed metric checks before rollback (default 10) + threshold: {{ .Values.rolloutStrategy.configs.threshold | default 10 }} + # max traffic percentage (0-100) routed to canary (default 30) + maxWeight: {{ .Values.rolloutStrategy.configs.maxWeight | default 30 }} + # canary increment step percentage (0-100) (default 10) + stepWeight: {{ .Values.rolloutStrategy.configs.stepWeight | default 10 }} + # metrics checks + metrics: + {{- toYaml .Values.rolloutStrategy.configs.metrics | nindent 6 }} + {{- if .Values.rolloutStrategy.configs.webhooks }} + # testing (optional) + webhooks: + {{- toYaml .Values.rolloutStrategy.configs.webhooks | nindent 6 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/base/templates/service.yaml b/charts/base/templates/service.yaml index 80311f7..7b07be0 100644 --- a/charts/base/templates/service.yaml +++ b/charts/base/templates/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.service.enabled -}} +{{- if and .Values.service.enabled (not (and .Values.rolloutStrategy.enabled (eq .Values.rolloutStrategy.operator "flagger") ) ) -}} apiVersion: v1 kind: Service metadata: diff --git a/charts/base/values.yaml b/charts/base/values.yaml index 6247629..f88e11d 100644 --- a/charts/base/values.yaml +++ b/charts/base/values.yaml @@ -49,7 +49,6 @@ job: serviceAccount: create: false annotations: {} - podAnnotations: {} @@ -237,3 +236,69 @@ serviceMonitor: interval: 30s targetPort: 80 path: /metrics + +# This config allows to enable custom rollout strategies by using different providers +# right now only flagger (https://flagger.app/) provider supported and tested for canary with nginx +## NOTEs for flagger provider: +## - flagger supports several service meshes and ingresses as provider for traffic splitting, and by default we have using nginx here, so you have to check docs and have at least one used for you app +## - you need to have flagger tool/operator already installed to be able to use its crd, this can be done by installing flagger helm https://artifacthub.io/packages/helm/flagger/flagger +## - also there is need to have at least one metric server/provider enabled(it supports) like prometheus as it uses metrics for checking success rates, the flagger helm allows to install prometheus +## - with flagger enabled we disable native kubernetes service as flagger creates/overrides this service +## - with separate installed prometheus operator(not one that comes with flagger helm) the default `request-success-rate` and `request-duration` metrics templates may not work so you may need to create custom metric templates, an example for canary+nginx+prometheus can be found here /examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml +rolloutStrategy: + enabled: false + operator: flagger + configs: {} + # example for flagger configs + # configs: + # progressDeadlineSeconds: 61 # the maximum time in seconds for the canary deployment to make progress before it is rollback (default 600s) + # canaryReadyThreshold: 51 # minimum percentage of canary pods that must be ready before considering canary ready for traffic shifting (default 100) + # primaryReadyThreshold: 51 # minimum percentage of primary pods that must be ready before considering primary ready for traffic shifting (default 100) + # interval: 11s # schedule interval (default 60s) + # threshold: 11 # max number of failed metric checks before rollback (default 10) + # maxWeight: 31 # max traffic percentage (0-100) routed to canary (default 30) + # stepWeight: 11 # canary increment step percentage (0-100) (default 10) + # # min and max replicas count for primary hpa, default to main app hpa, the main app hpa values also being used for canary deploy hpa so we use this options to have custom values for primary hpa + # primaryScalerMinReplicas: 3 + # primaryScalerMaxReplicas: 7 + # metrics: # metrics template configs to use for identifying if canary deploy handles request normally, the `request-success-rate` and `request-duration` named ones are available by default, and you can create custom metric templates + # - name: request-success-rate + # # minimum req success rate (non 5xx responses) percentage (0-100) + # thresholdRange: + # min: 99 + # interval: 1m + # - name: request-duration + # # maximum req duration P99, milliseconds + # thresholdRange: + # max: 500 + # interval: 1m + # # - name: request-success-rate-custom + # # interval: 1m + # # templateRef: + # # name: request-success-rate-custom + # # namespace: ingress-nginx + # # # minimum req success rate (non 5xx responses) percentage (0-100) + # # thresholdRange: + # # min: 99 + # # - name: request-duration-custom + # # interval: 1m + # # templateRef: + # # name: request-duration-custom + # # namespace: ingress-nginx + # # # maximum req duration P99, milliseconds + # # thresholdRange: + # # max: 500 + # # + # webhooks: # webhooks can be used for load testing before traffic switching to canaries by using `pre-rollout` type and also generating traffic + # - name: acceptance-test + # type: pre-rollout + # url: http://flagger-loadtester.localhost/ + # timeout: 30s + # metadata: + # type: bash + # cmd: "curl -sd 'test' http://http-echo-canary/ping | grep ping" + # - name: load-test + # url: http://flagger-loadtester.localhost/ + # timeout: 5s + # metadata: + # cmd: "hey -z 1m -q 3 -c 1 http://http-echo.localhost/ping" diff --git a/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml b/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml new file mode 100644 index 0000000..758025f --- /dev/null +++ b/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml @@ -0,0 +1,62 @@ +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: request-success-rate-custom + namespace: ingress-nginx +spec: + provider: + address: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090 + type: prometheus + query: | + sum( + rate( + nginx_ingress_controller_requests{ + exported_namespace="{{ namespace }}", + ingress="{{ ingress }}", + canary!="", + status!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + nginx_ingress_controller_requests{ + exported_namespace="{{ namespace }}", + ingress="{{ ingress }}", + canary!="" + }[{{ interval }}] + ) + ) + * 100 +--- +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: request-duration-custom + namespace: ingress-nginx +spec: + provider: + address: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090 + type: prometheus + query: | + sum( + rate( + nginx_ingress_controller_response_duration_seconds_sum{ + exported_namespace="{{ namespace }}", + ingress="{{ ingress }}", + canary!="" + }[{{ interval }}] + ) + ) + / + sum( + rate( + nginx_ingress_controller_response_duration_seconds_count{ + exported_namespace="{{ namespace }}", + ingress="{{ ingress }}", + canary!="" + }[{{ interval }}] + ) + ) + * 1000 From 07759a55b1bc25e4cb7a4e77403b6e412926cb26 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 14:12:43 +0400 Subject: [PATCH 02/10] feat(DMVP-5664): flagger canary deployment ability in base helm chart increase chart version --- charts/base/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/base/Chart.yaml b/charts/base/Chart.yaml index 06ebdc2..1def74e 100644 --- a/charts/base/Chart.yaml +++ b/charts/base/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.5 +version: 0.2.6 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.2.5" +appVersion: "0.2.6" From b603d246da0a57aca57790aba8ddaa0feea90ded Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 14:22:46 +0400 Subject: [PATCH 03/10] feat(DMVP-5664): fix comments --- charts/base/values.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/charts/base/values.yaml b/charts/base/values.yaml index f88e11d..7dae8c8 100644 --- a/charts/base/values.yaml +++ b/charts/base/values.yaml @@ -237,9 +237,9 @@ serviceMonitor: targetPort: 80 path: /metrics -# This config allows to enable custom rollout strategies by using different providers -# right now only flagger (https://flagger.app/) provider supported and tested for canary with nginx -## NOTEs for flagger provider: +# This config allows to enable custom rollout strategies by using different providers/operators +# right now only flagger (https://flagger.app/) operator supported and tested for canary with nginx +## NOTE for flagger operator: ## - flagger supports several service meshes and ingresses as provider for traffic splitting, and by default we have using nginx here, so you have to check docs and have at least one used for you app ## - you need to have flagger tool/operator already installed to be able to use its crd, this can be done by installing flagger helm https://artifacthub.io/packages/helm/flagger/flagger ## - also there is need to have at least one metric server/provider enabled(it supports) like prometheus as it uses metrics for checking success rates, the flagger helm allows to install prometheus @@ -249,8 +249,9 @@ rolloutStrategy: enabled: false operator: flagger configs: {} - # example for flagger configs + # here are all supported flagger configs # configs: + # provider: nginx # the flagger ingress/service-mesh provider (default nginx) # progressDeadlineSeconds: 61 # the maximum time in seconds for the canary deployment to make progress before it is rollback (default 600s) # canaryReadyThreshold: 51 # minimum percentage of canary pods that must be ready before considering canary ready for traffic shifting (default 100) # primaryReadyThreshold: 51 # minimum percentage of primary pods that must be ready before considering primary ready for traffic shifting (default 100) From 487a73c35403c30b6818ed5eecee0021c5cb25cd Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 14:25:51 +0400 Subject: [PATCH 04/10] feat(DMVP-5664): fix example flagger metric-template --- .../flagger-canary-nginx-custom.metric-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml b/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml index 758025f..7018391 100644 --- a/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml +++ b/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml @@ -5,7 +5,7 @@ metadata: namespace: ingress-nginx spec: provider: - address: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090 + address: http://prometheus-service.monitoring:9090 type: prometheus query: | sum( @@ -37,7 +37,7 @@ metadata: namespace: ingress-nginx spec: provider: - address: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090 + address: http://prometheus-service.monitoring:9090 type: prometheus query: | sum( From 34692d9e0037714b80ab1b745a661120ab64c745 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 14:29:54 +0400 Subject: [PATCH 05/10] feat(DMVP-5664): flagger canary deployment ability in base helm chart increase chart version --- charts/base/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/base/Chart.yaml b/charts/base/Chart.yaml index 1def74e..51062ae 100644 --- a/charts/base/Chart.yaml +++ b/charts/base/Chart.yaml @@ -15,10 +15,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.6 +version: 0.2.7 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.2.6" +appVersion: "0.2.7" From b1b6b348a7ed71db2d349210d47756acc5bbc039 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 15:06:44 +0400 Subject: [PATCH 06/10] feat(DMVP-5664): have rolloutStrategy option description/example in README.md also --- charts/base/README.md | 68 +++++++++++++++++++++++++++++++++++++++++ charts/base/values.yaml | 4 +-- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/charts/base/README.md b/charts/base/README.md index a72a6fa..b1d7985 100644 --- a/charts/base/README.md +++ b/charts/base/README.md @@ -602,3 +602,71 @@ terminationGracePeriodSeconds: 65 ### Deployment use chart-hooks annotations: "helm.sh/hook": pre-install,pre-upgrade + + +### custom rollout strategy(canary,blue/gree) configs by using flagger +```yaml +# This config allows to enable custom rollout strategies by using different providers/operators +# right now only flagger (https://flagger.app/) operator supported and tested for canary with nginx +## NOTE for flagger operator: +## - flagger supports several service meshes and ingresses as provider for traffic splitting, and by default we have using nginx here, so you have to check docs and have at least one used for you app +## - you need to have flagger tool/operator already installed to be able to use its crd, this can be done by installing flagger helm https://artifacthub.io/packages/helm/flagger/flagger +## - also there is need to have at least one metric server/provider enabled(it supports) like prometheus as it uses metrics for checking success rates, the flagger helm allows to install prometheus +## - with flagger enabled we disable native kubernetes service as flagger creates/overrides this service +## - with separate installed prometheus operator(not one that comes with flagger helm) the default `request-success-rate` and `request-duration` metrics templates may not work so you may need to create custom metric templates, an example for canary+nginx+prometheus can be found here /examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml +rolloutStrategy: + enabled: true + operator: flagger + configs: # here are all supported flagger configs + provider: nginx # the flagger ingress/service-mesh provider (default nginx) + progressDeadlineSeconds: 61 # the maximum time in seconds for the canary deployment to make progress before it is rollback (default 600s) + canaryReadyThreshold: 51 # minimum percentage of canary pods that must be ready before considering canary ready for traffic shifting (default 100) + primaryReadyThreshold: 51 # minimum percentage of primary pods that must be ready before considering primary ready for traffic shifting (default 100) + interval: 11s # schedule interval (default 60s) + threshold: 11 # max number of failed metric checks before rollback (default 10) + maxWeight: 31 # max traffic percentage (0-100) routed to canary (default 30) + stepWeight: 11 # canary increment step percentage (0-100) (default 10) + # min and max replicas count for primary hpa, default to main app hpa, the main app hpa values also being used for canary deploy hpa so we use this options to have custom values for primary hpa + primaryScalerMinReplicas: 3 + primaryScalerMaxReplicas: 7 + metrics: # metrics template configs to use for identifying if canary deploy handles request normally, the `request-success-rate` and `request-duration` named ones are available by default, and you can create custom metric templates + - name: request-success-rate + # minimum req success rate (non 5xx responses) percentage (0-100) + thresholdRange: + min: 99 + interval: 1m + - name: request-duration + # maximum req duration P99, milliseconds + thresholdRange: + max: 500 + interval: 1m + # - name: request-success-rate-custom + # interval: 1m + # templateRef: + # name: request-success-rate-custom + # namespace: ingress-nginx + # # minimum req success rate (non 5xx responses) percentage (0-100) + # thresholdRange: + # min: 99 + # - name: request-duration-custom + # interval: 1m + # templateRef: + # name: request-duration-custom + # namespace: ingress-nginx + # # maximum req duration P99, milliseconds + # thresholdRange: + # max: 500 + webhooks: # (optional) webhooks can be used for load testing before traffic switching to canaries by using `pre-rollout` type and also generating traffic + - name: acceptance-test + type: pre-rollout + url: http://flagger-loadtester.localhost/ + timeout: 30s + metadata: + type: bash + cmd: "curl -sd 'test' http://http-echo-canary/ping | grep ping" + - name: load-test + url: http://flagger-loadtester.localhost/ + timeout: 5s + metadata: + cmd: "hey -z 1m -q 3 -c 1 http://http-echo.localhost/ping" +``` diff --git a/charts/base/values.yaml b/charts/base/values.yaml index 7dae8c8..9d916a8 100644 --- a/charts/base/values.yaml +++ b/charts/base/values.yaml @@ -289,8 +289,8 @@ rolloutStrategy: # # # maximum req duration P99, milliseconds # # thresholdRange: # # max: 500 - # # - # webhooks: # webhooks can be used for load testing before traffic switching to canaries by using `pre-rollout` type and also generating traffic + # + # webhooks: # (optional) webhooks can be used for load testing before traffic switching to canaries by using `pre-rollout` type and also generating traffic # - name: acceptance-test # type: pre-rollout # url: http://flagger-loadtester.localhost/ From 51aada2fb48d0218d91bf5ceb839d4c8c511e6ec Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 17:07:39 +0400 Subject: [PATCH 07/10] feat(DMVP-5664): have separate flagger-metric-template chart for flagger metric template creation --- charts/base/README.md | 2 +- charts/base/values.yaml | 2 +- charts/flagger-metric-template/.helmignore | 23 +++++++ charts/flagger-metric-template/Chart.yaml | 24 ++++++++ .../files/nginx-request-success-rate-custom | 0 ...r-canary-nginx-custom.metric-template.yaml | 60 +++++++++++++++++++ .../templates/metric-templates.yaml | 11 ++++ charts/flagger-metric-template/values.yaml | 19 ++++++ 8 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 charts/flagger-metric-template/.helmignore create mode 100644 charts/flagger-metric-template/Chart.yaml create mode 100644 charts/flagger-metric-template/files/nginx-request-success-rate-custom create mode 100644 charts/flagger-metric-template/templates/flagger-canary-nginx-custom.metric-template.yaml create mode 100644 charts/flagger-metric-template/templates/metric-templates.yaml create mode 100644 charts/flagger-metric-template/values.yaml diff --git a/charts/base/README.md b/charts/base/README.md index b1d7985..2bfab95 100644 --- a/charts/base/README.md +++ b/charts/base/README.md @@ -613,7 +613,7 @@ annotations: ## - you need to have flagger tool/operator already installed to be able to use its crd, this can be done by installing flagger helm https://artifacthub.io/packages/helm/flagger/flagger ## - also there is need to have at least one metric server/provider enabled(it supports) like prometheus as it uses metrics for checking success rates, the flagger helm allows to install prometheus ## - with flagger enabled we disable native kubernetes service as flagger creates/overrides this service -## - with separate installed prometheus operator(not one that comes with flagger helm) the default `request-success-rate` and `request-duration` metrics templates may not work so you may need to create custom metric templates, an example for canary+nginx+prometheus can be found here /examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml +## - with separate installed prometheus operator(not one that comes with flagger helm) the default `request-success-rate` and `request-duration` metrics templates may not work so you may need to create custom metric templates, the canary+nginx+prometheus metric template can be created by using `dasmeta/flagger-metric-template` chart rolloutStrategy: enabled: true operator: flagger diff --git a/charts/base/values.yaml b/charts/base/values.yaml index 9d916a8..3fb648c 100644 --- a/charts/base/values.yaml +++ b/charts/base/values.yaml @@ -244,7 +244,7 @@ serviceMonitor: ## - you need to have flagger tool/operator already installed to be able to use its crd, this can be done by installing flagger helm https://artifacthub.io/packages/helm/flagger/flagger ## - also there is need to have at least one metric server/provider enabled(it supports) like prometheus as it uses metrics for checking success rates, the flagger helm allows to install prometheus ## - with flagger enabled we disable native kubernetes service as flagger creates/overrides this service -## - with separate installed prometheus operator(not one that comes with flagger helm) the default `request-success-rate` and `request-duration` metrics templates may not work so you may need to create custom metric templates, an example for canary+nginx+prometheus can be found here /examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml +## - with separate installed prometheus operator(not one that comes with flagger helm) the default `request-success-rate` and `request-duration` metrics templates may not work so you may need to create custom metric templates, the canary+nginx+prometheus metric template can be created by using `dasmeta/flagger-metric-template` chart rolloutStrategy: enabled: false operator: flagger diff --git a/charts/flagger-metric-template/.helmignore b/charts/flagger-metric-template/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/flagger-metric-template/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/flagger-metric-template/Chart.yaml b/charts/flagger-metric-template/Chart.yaml new file mode 100644 index 0000000..499a688 --- /dev/null +++ b/charts/flagger-metric-template/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: flagger-metric-template +description: A Helm chart for Kubernetes to create Flagger metric templates + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/charts/flagger-metric-template/files/nginx-request-success-rate-custom b/charts/flagger-metric-template/files/nginx-request-success-rate-custom new file mode 100644 index 0000000..e69de29 diff --git a/charts/flagger-metric-template/templates/flagger-canary-nginx-custom.metric-template.yaml b/charts/flagger-metric-template/templates/flagger-canary-nginx-custom.metric-template.yaml new file mode 100644 index 0000000..56832ec --- /dev/null +++ b/charts/flagger-metric-template/templates/flagger-canary-nginx-custom.metric-template.yaml @@ -0,0 +1,60 @@ +{{- if .Values.createNginxCustomMetricTemplates -}} +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: request-success-rate-custom +spec: + provider: + {{- toYaml .Values.provider | nindent 4 }} + query: | + sum( + rate( + nginx_ingress_controller_requests{ + exported_namespace="{{ "{{" }} namespace {{ "}}" }}", + ingress="{{ "{{" }} ingress {{ "}}" }}", + canary!="", + status!~"5.*" + }[{{ "{{" }} interval {{ "}}" }}] + ) + ) + / + sum( + rate( + nginx_ingress_controller_requests{ + exported_namespace="{{ "{{" }} namespace {{ "}}" }}", + ingress="{{ "{{" }} ingress {{ "}}" }}", + canary!="" + }[{{ "{{" }} interval {{ "}}" }}] + ) + ) + * 100 +--- +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: request-duration-custom +spec: + provider: + {{- toYaml .Values.provider | nindent 4 }} + query: | + sum( + rate( + nginx_ingress_controller_response_duration_seconds_sum{ + exported_namespace="{{ "{{" }} namespace {{ "}}" }}", + ingress="{{ "{{" }} ingress {{ "}}" }}", + canary!="" + }[{{ "{{" }} interval {{ "}}" }}] + ) + ) + / + sum( + rate( + nginx_ingress_controller_response_duration_seconds_count{ + exported_namespace="{{ "{{" }} namespace {{ "}}" }}", + ingress="{{ "{{" }} ingress {{ "}}" }}", + canary!="" + }[{{ "{{" }} interval {{ "}}" }}] + ) + ) + * 1000 +{{- end }} diff --git a/charts/flagger-metric-template/templates/metric-templates.yaml b/charts/flagger-metric-template/templates/metric-templates.yaml new file mode 100644 index 0000000..792f475 --- /dev/null +++ b/charts/flagger-metric-template/templates/metric-templates.yaml @@ -0,0 +1,11 @@ +{{- range $key, $metricTemplate := .Values.metricTemplates }} +--- +apiVersion: flagger.app/v1beta1 +kind: MetricTemplate +metadata: + name: {{ $metricTemplate.name }} +spec: + provider: + {{- toYaml $.Values.provider | nindent 4 }} + query: {{ $metricTemplate.query }} +{{- end }} diff --git a/charts/flagger-metric-template/values.yaml b/charts/flagger-metric-template/values.yaml new file mode 100644 index 0000000..1664070 --- /dev/null +++ b/charts/flagger-metric-template/values.yaml @@ -0,0 +1,19 @@ +# Default values for flagger-metric-template. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# metrics provider config for metric-templates, for more info check doc https://docs.flagger.app/usage/metrics#custom-metrics +provider: + type: prometheus + address: http://prometheus-service.monitoring:9090 + +# Whether to create `request-success-rate-custom` and `request-duration-custom` nginx metric templates, this can be used in case we use custom prometheus operator instead of flagger helm included one +createNginxCustomMetricTemplates: true + +# List of custom metric template configs +# metricTemplates: [] +## example for metric templates +metricTemplates: + - name: my-custom-request-rate-metric-template + query: | + sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}",status!~"5.*"}[1m]))/sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}"}[1m]))*100 From e46dad517dc780b4cce5cbf5f7f647254fef5d24 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 17:08:22 +0400 Subject: [PATCH 08/10] feat(DMVP-5664): remove example for flagger metric template as we have chart for it --- ...r-canary-nginx-custom.metric-template.yaml | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml diff --git a/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml b/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml deleted file mode 100644 index 7018391..0000000 --- a/examples/kubectl-manifests/flagger-canary-nginx-custom.metric-template.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: flagger.app/v1beta1 -kind: MetricTemplate -metadata: - name: request-success-rate-custom - namespace: ingress-nginx -spec: - provider: - address: http://prometheus-service.monitoring:9090 - type: prometheus - query: | - sum( - rate( - nginx_ingress_controller_requests{ - exported_namespace="{{ namespace }}", - ingress="{{ ingress }}", - canary!="", - status!~"5.*" - }[{{ interval }}] - ) - ) - / - sum( - rate( - nginx_ingress_controller_requests{ - exported_namespace="{{ namespace }}", - ingress="{{ ingress }}", - canary!="" - }[{{ interval }}] - ) - ) - * 100 ---- -apiVersion: flagger.app/v1beta1 -kind: MetricTemplate -metadata: - name: request-duration-custom - namespace: ingress-nginx -spec: - provider: - address: http://prometheus-service.monitoring:9090 - type: prometheus - query: | - sum( - rate( - nginx_ingress_controller_response_duration_seconds_sum{ - exported_namespace="{{ namespace }}", - ingress="{{ ingress }}", - canary!="" - }[{{ interval }}] - ) - ) - / - sum( - rate( - nginx_ingress_controller_response_duration_seconds_count{ - exported_namespace="{{ namespace }}", - ingress="{{ ingress }}", - canary!="" - }[{{ interval }}] - ) - ) - * 1000 From 0877f1afd0c1dfde079bffc5a2aa8f1204e302e8 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 17:46:19 +0400 Subject: [PATCH 09/10] feat(DMVP-5664): fix flagger-metric-template chart default options, have readme and do cleanup --- charts/flagger-metric-template/README.md | 12 ++++++++++++ .../files/nginx-request-success-rate-custom | 0 charts/flagger-metric-template/values.yaml | 10 +++++----- 3 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 charts/flagger-metric-template/README.md delete mode 100644 charts/flagger-metric-template/files/nginx-request-success-rate-custom diff --git a/charts/flagger-metric-template/README.md b/charts/flagger-metric-template/README.md new file mode 100644 index 0000000..3453505 --- /dev/null +++ b/charts/flagger-metric-template/README.md @@ -0,0 +1,12 @@ +# This helm chart allows to create flagger custom metric templates to use in canary rollout + +## There is option named `createNginxCustomMetricTemplates`(which is true by default) to create nginx custom metrics named `request-success-rate-custom` and `request-duration-custom` + +## example of custom metric templates +```yaml +metricTemplates: + - name: my-custom-request-rate-metric-template + query: | + sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}",status!~"5.*"}[1m]))/sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}"}[1m]))*100 + +``` \ No newline at end of file diff --git a/charts/flagger-metric-template/files/nginx-request-success-rate-custom b/charts/flagger-metric-template/files/nginx-request-success-rate-custom deleted file mode 100644 index e69de29..0000000 diff --git a/charts/flagger-metric-template/values.yaml b/charts/flagger-metric-template/values.yaml index 1664070..ee57bbb 100644 --- a/charts/flagger-metric-template/values.yaml +++ b/charts/flagger-metric-template/values.yaml @@ -11,9 +11,9 @@ provider: createNginxCustomMetricTemplates: true # List of custom metric template configs -# metricTemplates: [] +metricTemplates: [] ## example for metric templates -metricTemplates: - - name: my-custom-request-rate-metric-template - query: | - sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}",status!~"5.*"}[1m]))/sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}"}[1m]))*100 +# metricTemplates: +# - name: my-custom-request-rate-metric-template +# query: | +# sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}",status!~"5.*"}[1m]))/sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}"}[1m]))*100 From d1728c23893e72b8f32e56454de4887645247832 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Wed, 6 Nov 2024 17:48:43 +0400 Subject: [PATCH 10/10] feat(DMVP-5664): have new line in the end of file --- charts/flagger-metric-template/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/charts/flagger-metric-template/README.md b/charts/flagger-metric-template/README.md index 3453505..e6842f8 100644 --- a/charts/flagger-metric-template/README.md +++ b/charts/flagger-metric-template/README.md @@ -8,5 +8,4 @@ metricTemplates: - name: my-custom-request-rate-metric-template query: | sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}",status!~"5.*"}[1m]))/sum(rate(nginx_ingress_controller_requests{exported_namespace="{{ namespace }}",ingress="{{ ingress }}"}[1m]))*100 - -``` \ No newline at end of file +```