From 2de58bf5fdd0fe4ea88655ce2034b2b22d3d8502 Mon Sep 17 00:00:00 2001 From: Tigran Muradyan Date: Fri, 22 Nov 2024 18:04:31 +0400 Subject: [PATCH] feat(DMVP-5592): have new karpenter-nodes chart to create karpenter node resources and other k8s resources --- charts/karpenter-nodes/.helmignore | 23 ++++++ charts/karpenter-nodes/Chart.yaml | 24 +++++++ charts/karpenter-nodes/README.md | 72 +++++++++++++++++++ charts/karpenter-nodes/templates/NOTES.txt | 4 ++ .../templates/ec2-node-classes.yaml | 9 +++ .../templates/flow-schemas.yaml | 61 ++++++++++++++++ .../karpenter-nodes/templates/node-pools.yaml | 9 +++ charts/karpenter-nodes/values.yaml | 72 +++++++++++++++++++ 8 files changed, 274 insertions(+) create mode 100644 charts/karpenter-nodes/.helmignore create mode 100644 charts/karpenter-nodes/Chart.yaml create mode 100644 charts/karpenter-nodes/README.md create mode 100644 charts/karpenter-nodes/templates/NOTES.txt create mode 100644 charts/karpenter-nodes/templates/ec2-node-classes.yaml create mode 100644 charts/karpenter-nodes/templates/flow-schemas.yaml create mode 100644 charts/karpenter-nodes/templates/node-pools.yaml create mode 100644 charts/karpenter-nodes/values.yaml diff --git a/charts/karpenter-nodes/.helmignore b/charts/karpenter-nodes/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/karpenter-nodes/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/karpenter-nodes/Chart.yaml b/charts/karpenter-nodes/Chart.yaml new file mode 100644 index 0000000..297ebce --- /dev/null +++ b/charts/karpenter-nodes/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: karpenter-integration +description: A Helm chart for Kubernetes to create configure and create karpenter crd resources(it is supposed that karpenter operator chart with its crds have been already installed) + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.1.0" diff --git a/charts/karpenter-nodes/README.md b/charts/karpenter-nodes/README.md new file mode 100644 index 0000000..058acc9 --- /dev/null +++ b/charts/karpenter-nodes/README.md @@ -0,0 +1,72 @@ +# This helm chart allows to create karpenter EC2NodeClass and NodePool resources check here for more details https://karpenter.sh/docs/concepts/ , it also creates FlowSchema resources(in case if karpenter was created not in kube-system, in case of karpenter installed on kube-system the kubernetes provides those resource automatically) for karpenter to access kubernetes api server with priority + +## to install the chart use the command +```sh +helm upgrade --install -n karpenter karpenter-nodes dasmeta/karpenter-nodes -f path-of-values.yaml +``` + +## example of configs to create karpenter nodes resources +```yaml +# EC2NodeClass object configs, to enable AWS specific settings, each NodePool must reference an NodeClass, for more info look https://karpenter.sh/docs/concepts/nodeclasses/ +ec2NodeClasses: + my-node-class: + amiFamily: AL2 + amiSelectorTerms: # aws ami which will be used for nodes + - id: ami-0e7df911d76024f90 + role: # iam identity role name nodes should assume (optional) + securityGroupSelectorTerms: # vpc security group nodes should get, usually this is same group used/created for eks standard nodes + - tags: + karpenter.sh/discovery: test-cluster-with-karpenter + subnetSelectorTerms: # the vpc subnets used for nodes, usually this is same list that used for eks + - id: subnet- + - id: subnet- + - id: subnet- +# NodePool to create, this is map of :, for more info look https://karpenter.sh/docs/concepts/nodepools/ +nodePools: + my-node-pool: + template: + spec: + expireAfter: Never + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: my-node-class # this is name of EC2NodeClass, an node-class can be referenced by multiple node-pools + requirements: + - key: karpenter.k8s.aws/instance-cpu + operator: Lt + values: + - "5" + - key: karpenter.k8s.aws/instance-cpu + operator: Gt + values: + - "1" + - key: karpenter.k8s.aws/instance-memory + operator: Lt + values: + - "90000" + - key: karpenter.k8s.aws/instance-memory + operator: Gt + values: + - "1000" + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: + - "2" + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.sh/capacity-type + operator: In + values: + - spot + - on-demand + disruption: + budgets: + - nodes: 10% + consolidateAfter: 1m + consolidationPolicy: WhenEmptyOrUnderutilized + limits: + cpu: 10 + weight: 1 +``` diff --git a/charts/karpenter-nodes/templates/NOTES.txt b/charts/karpenter-nodes/templates/NOTES.txt new file mode 100644 index 0000000..1d0f4f8 --- /dev/null +++ b/charts/karpenter-nodes/templates/NOTES.txt @@ -0,0 +1,4 @@ +To check/get created/updated resources run: +kubectl get FlowSchema -n {{ .Release.Namespace }} +kubectl get EC2NodeClass +kubectl get NodePool diff --git a/charts/karpenter-nodes/templates/ec2-node-classes.yaml b/charts/karpenter-nodes/templates/ec2-node-classes.yaml new file mode 100644 index 0000000..ec7ab7d --- /dev/null +++ b/charts/karpenter-nodes/templates/ec2-node-classes.yaml @@ -0,0 +1,9 @@ +{{- range $name, $ec2NodeClassSpec := .Values.ec2NodeClasses }} +--- +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: {{ $name }} +spec: + {{- toYaml $ec2NodeClassSpec | nindent 2 }} +{{- end }} diff --git a/charts/karpenter-nodes/templates/flow-schemas.yaml b/charts/karpenter-nodes/templates/flow-schemas.yaml new file mode 100644 index 0000000..eab0eee --- /dev/null +++ b/charts/karpenter-nodes/templates/flow-schemas.yaml @@ -0,0 +1,61 @@ +{{- if and .Values.createFlowSchemas (ne .Values.karpenterNamespace "kube-system") -}} +apiVersion: flowcontrol.apiserver.k8s.io/v1 +kind: FlowSchema +metadata: + name: karpenter-leader-election +spec: + distinguisherMethod: + type: ByUser + matchingPrecedence: 200 + priorityLevelConfiguration: + name: leader-election + rules: + - resourceRules: + - apiGroups: + - coordination.k8s.io + namespaces: + - '*' + resources: + - leases + verbs: + - get + - create + - update + subjects: + - kind: ServiceAccount + serviceAccount: + name: {{ .Values.karpenterServiceAccount }} + namespace: {{ .Values.karpenterNamespace }} +--- +apiVersion: flowcontrol.apiserver.k8s.io/v1 +kind: FlowSchema +metadata: + name: karpenter-workload +spec: + distinguisherMethod: + type: ByUser + matchingPrecedence: 1000 + priorityLevelConfiguration: + name: workload-high + rules: + - nonResourceRules: + - nonResourceURLs: + - '*' + verbs: + - '*' + resourceRules: + - apiGroups: + - '*' + clusterScope: true + namespaces: + - '*' + resources: + - '*' + verbs: + - '*' + subjects: + - kind: ServiceAccount + serviceAccount: + name: {{ .Values.karpenterServiceAccount }} + namespace: {{ .Values.karpenterNamespace }} +{{- end }} diff --git a/charts/karpenter-nodes/templates/node-pools.yaml b/charts/karpenter-nodes/templates/node-pools.yaml new file mode 100644 index 0000000..3f27b96 --- /dev/null +++ b/charts/karpenter-nodes/templates/node-pools.yaml @@ -0,0 +1,9 @@ +{{- range $name, $nodePoolSpec := .Values.nodePools }} +--- +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: {{ $name }} +spec: + {{- toYaml $nodePoolSpec | nindent 2 }} +{{- end }} diff --git a/charts/karpenter-nodes/values.yaml b/charts/karpenter-nodes/values.yaml new file mode 100644 index 0000000..7731560 --- /dev/null +++ b/charts/karpenter-nodes/values.yaml @@ -0,0 +1,72 @@ +# Default values for karpenter-integration. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +createFlowSchemas: true # creates karpenter namespace FlowSchema resources, it is needed (for putting karpenter into higher-priority FlowSchemas) in case we have karpenter installed in a different namespace than the default "kube-system", for more info look https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#preventing-apiserver-request-throttling +karpenterServiceAccount: karpenter # service account name in karpenter operator namespace which attached/used by karpenter pods for cloud operations authentication +karpenterNamespace: karpenter # the namespace where karpenter operator/helm have been installed + +ec2NodeClasses: {} # EC2NodeClass object configs, to enable AWS specific settings, each NodePool must reference an NodeClass, for more info look https://karpenter.sh/docs/concepts/nodeclasses/ +## example of how to create EC2NodeClass resources +# ec2NodeClasses: +# my-node-class: +# amiFamily: AL2 +# amiSelectorTerms: # aws ami which will be used for nodes +# - id: ami-0e7df911d76024f90 +# role: # iam identity role name nodes should assume (optional) +# securityGroupSelectorTerms: # vpc security group nodes should get, usually this is same group used/created for eks standard nodes +# - tags: +# karpenter.sh/discovery: test-cluster-with-karpenter +# subnetSelectorTerms: # the vpc subnets used for nodes, usually this is same list that used for eks +# - id: subnet- +# - id: subnet- +# - id: subnet- +nodePools: {} # NodePool to create, this is map of :, for more info look https://karpenter.sh/docs/concepts/nodepools/ +## example of how to create NodePool resources +# nodePools: +# my-node-pool: +# template: +# spec: +# expireAfter: Never +# nodeClassRef: +# group: karpenter.k8s.aws +# kind: EC2NodeClass +# name: my-node-class # this is name of EC2NodeClass, an node-class can be referenced by multiple node-pools +# requirements: +# - key: karpenter.k8s.aws/instance-cpu +# operator: Lt +# values: +# - "5" +# - key: karpenter.k8s.aws/instance-cpu +# operator: Gt +# values: +# - "1" +# - key: karpenter.k8s.aws/instance-memory +# operator: Lt +# values: +# - "90000" +# - key: karpenter.k8s.aws/instance-memory +# operator: Gt +# values: +# - "1000" +# - key: karpenter.k8s.aws/instance-generation +# operator: Gt +# values: +# - "2" +# - key: kubernetes.io/arch +# operator: In +# values: +# - amd64 +# - key: karpenter.sh/capacity-type +# operator: In +# values: +# - spot +# - on-demand +# disruption: +# budgets: +# - nodes: 10% +# consolidateAfter: 1m +# consolidationPolicy: WhenEmptyOrUnderutilized +# limits: +# cpu: 10 +# weight: 1