From 6dacd63ebd1102b095a5a6f363b3e7daa3e28d87 Mon Sep 17 00:00:00 2001 From: Frame Date: Tue, 27 Feb 2024 19:49:34 +0800 Subject: [PATCH] Add chart v1.4.1 (#86) Signed-off-by: saintube --- versions/v1.4.1/.helmignore | 23 + versions/v1.4.1/Chart.yaml | 14 + versions/v1.4.1/README.md | 88 + versions/v1.4.1/templates/_helpers.tpl | 97 + ...rdinator.sh_clustercolocationprofiles.yaml | 212 +++ ...a.koordinator.sh_elasticquotaprofiles.yaml | 106 ++ .../scheduling.koordinator.sh_devices.yaml | 172 ++ ...uling.koordinator.sh_podmigrationjobs.yaml | 522 ++++++ ...cheduling.koordinator.sh_reservations.yaml | 356 ++++ .../scheduling.sigs.k8s.io_elasticquotas.yaml | 89 + .../crd/scheduling.sigs.k8s.io_podgroups.yaml | 108 ++ .../crd/slo.koordinator.sh_nodemetrics.yaml | 1009 +++++++++++ .../crd/slo.koordinator.sh_nodeslos.yaml | 1601 +++++++++++++++++ ...gy.node.k8s.io_noderesourcetopologies.yaml | 147 ++ .../templates/koord-descheduler-config.yaml | 61 + .../v1.4.1/templates/koord-descheduler.yaml | 83 + versions/v1.4.1/templates/koord-manager.yaml | 130 ++ .../templates/koord-scheduler-config.yaml | 132 ++ .../v1.4.1/templates/koord-scheduler.yaml | 93 + .../v1.4.1/templates/koordlet-service.yaml | 20 + .../templates/koordlet-servicemonitor.yaml | 27 + versions/v1.4.1/templates/koordlet.yaml | 160 ++ .../templates/priority/koord-batch.yaml | 7 + .../v1.4.1/templates/priority/koord-free.yaml | 7 + .../v1.4.1/templates/priority/koord-mid.yaml | 7 + .../v1.4.1/templates/priority/koord-prod.yaml | 7 + .../templates/rbac/koord-descheduler.yaml | 113 ++ .../v1.4.1/templates/rbac/koord-manager.yaml | 254 +++ .../templates/rbac/koord-scheduler.yaml | 104 ++ versions/v1.4.1/templates/rbac/koordlet.yaml | 69 + .../templates/slo-controller-config.yaml | 16 + .../templates/webhookconfiguration.yaml | 240 +++ versions/v1.4.1/values.yaml | 151 ++ 33 files changed, 6225 insertions(+) create mode 100644 versions/v1.4.1/.helmignore create mode 100644 versions/v1.4.1/Chart.yaml create mode 100644 versions/v1.4.1/README.md create mode 100644 versions/v1.4.1/templates/_helpers.tpl create mode 100644 versions/v1.4.1/templates/crd/config.koordinator.sh_clustercolocationprofiles.yaml create mode 100644 versions/v1.4.1/templates/crd/quota.koordinator.sh_elasticquotaprofiles.yaml create mode 100644 versions/v1.4.1/templates/crd/scheduling.koordinator.sh_devices.yaml create mode 100644 versions/v1.4.1/templates/crd/scheduling.koordinator.sh_podmigrationjobs.yaml create mode 100644 versions/v1.4.1/templates/crd/scheduling.koordinator.sh_reservations.yaml create mode 100644 versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_elasticquotas.yaml create mode 100644 versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_podgroups.yaml create mode 100644 versions/v1.4.1/templates/crd/slo.koordinator.sh_nodemetrics.yaml create mode 100644 versions/v1.4.1/templates/crd/slo.koordinator.sh_nodeslos.yaml create mode 100644 versions/v1.4.1/templates/crd/topology.node.k8s.io_noderesourcetopologies.yaml create mode 100644 versions/v1.4.1/templates/koord-descheduler-config.yaml create mode 100644 versions/v1.4.1/templates/koord-descheduler.yaml create mode 100644 versions/v1.4.1/templates/koord-manager.yaml create mode 100644 versions/v1.4.1/templates/koord-scheduler-config.yaml create mode 100644 versions/v1.4.1/templates/koord-scheduler.yaml create mode 100644 versions/v1.4.1/templates/koordlet-service.yaml create mode 100644 versions/v1.4.1/templates/koordlet-servicemonitor.yaml create mode 100644 versions/v1.4.1/templates/koordlet.yaml create mode 100644 versions/v1.4.1/templates/priority/koord-batch.yaml create mode 100644 versions/v1.4.1/templates/priority/koord-free.yaml create mode 100644 versions/v1.4.1/templates/priority/koord-mid.yaml create mode 100644 versions/v1.4.1/templates/priority/koord-prod.yaml create mode 100644 versions/v1.4.1/templates/rbac/koord-descheduler.yaml create mode 100644 versions/v1.4.1/templates/rbac/koord-manager.yaml create mode 100644 versions/v1.4.1/templates/rbac/koord-scheduler.yaml create mode 100644 versions/v1.4.1/templates/rbac/koordlet.yaml create mode 100644 versions/v1.4.1/templates/slo-controller-config.yaml create mode 100644 versions/v1.4.1/templates/webhookconfiguration.yaml create mode 100644 versions/v1.4.1/values.yaml diff --git a/versions/v1.4.1/.helmignore b/versions/v1.4.1/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/versions/v1.4.1/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/versions/v1.4.1/Chart.yaml b/versions/v1.4.1/Chart.yaml new file mode 100644 index 0000000..6fbcf03 --- /dev/null +++ b/versions/v1.4.1/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +name: koordinator +description: A Helm chart for Koordinator +type: application +version: 1.4.1 +appVersion: 1.4.1 +icon: https://koordinator.sh/img/logo.svg +keywords: + - koordinator + - co-location + - mixed-workloads +home: https://koordinator.sh +sources: + - https://github.com/koordinator-sh/koordinator diff --git a/versions/v1.4.1/README.md b/versions/v1.4.1/README.md new file mode 100644 index 0000000..8bbe198 --- /dev/null +++ b/versions/v1.4.1/README.md @@ -0,0 +1,88 @@ +# Koordinator v1.4.1 + +## Configuration + +Note that installing this chart directly means it will use the default template values for Koordinator. + +You may have to set your specific configurations if it is deployed into a production cluster, or you want to configure feature-gates. + +### Optional: chart parameters + +The following table lists the configurable parameters of the chart and their default values. + +| Parameter | Description | Default | +| ----------------------------------------- | ---------------------------------------------------------------- |---------------------------------| +| `featureGates` | Feature gates for Koordinator, empty string means all by default | ` ` | +| `installation.namespace` | namespace for Koordinator installation | `koordinator-system` | +| `installation.createNamespace` | Whether to create the installation.namespace | `true` | +| `imageRepositoryHost` | Image repository host | `ghcr.io` | +| `manager.log.level` | Log level that koord-manager printed | `4` | +| `manager.replicas` | Replicas of koord-manager deployment | `2` | +| `manager.image.repository` | Repository for koord-manager image | `koordinatorsh/koord-manager` | +| `manager.image.tag` | Tag for koord-manager image | `v1.4.1` | +| `manager.resources.limits.cpu` | CPU resource limit of koord-manager container | `1000m` | +| `manager.resources.limits.memory` | Memory resource limit of koord-manager container | `1Gi` | +| `manager.resources.requests.cpu` | CPU resource request of koord-manager container | `500m` | +| `manager.resources.requests.memory` | Memory resource request of koord-manager container | `256Mi` | +| `manager.metrics.port` | Port of metrics served | `8080` | +| `manager.webhook.port` | Port of webhook served | `9443` | +| `manager.nodeAffinity` | Node affinity policy for koord-manager pod | `{}` | +| `manager.nodeSelector` | Node labels for koord-manager pod | `{}` | +| `manager.tolerations` | Tolerations for koord-manager pod | `[]` | +| `manager.resyncPeriod` | Resync period of informer koord-manager, defaults no resync | `0` | +| `manager.hostNetwork` | Whether koord-manager pod should run with hostnetwork | `false` | +| `scheduler.log.level` | Log level that koord-scheduler printed | `4` | +| `scheduler.replicas` | Replicas of koord-scheduler deployment | `2` | +| `scheduler.image.repository` | Repository for koord-scheduler image | `koordinatorsh/koord-scheduler` | +| `scheduler.image.tag` | Tag for koord-scheduler image | `v1.4.1` | +| `scheduler.resources.limits.cpu` | CPU resource limit of koord-scheduler container | `1000m` | +| `scheduler.resources.limits.memory` | Memory resource limit of koord-scheduler container | `1Gi` | +| `scheduler.resources.requests.cpu` | CPU resource request of koord-scheduler container | `500m` | +| `scheduler.resources.requests.memory` | Memory resource request of koord-scheduler container | `256Mi` | +| `scheduler.port` | Port of metrics served | `10251` | +| `scheduler.nodeAffinity` | Node affinity policy for koord-scheduler pod | `{}` | +| `scheduler.nodeSelector` | Node labels for koord-scheduler pod | `{}` | +| `scheduler.tolerations` | Tolerations for koord-scheduler pod | `[]` | +| `scheduler.hostNetwork` | Whether koord-scheduler pod should run with hostnetwork | `false` | +| `koordlet.log.level` | Log level that koordlet printed | `4` | +| `koordlet.image.repository` | Repository for koordlet image | `koordinatorsh/koordlet` | +| `koordlet.image.tag` | Tag for koordlet image | `v1.4.1` | +| `koordlet.resources.limits.cpu` | CPU resource limit of koordlet container | `500m` | +| `koordlet.resources.limits.memory` | Memory resource limit of koordlet container | `256Mi` | +| `koordlet.resources.requests.cpu` | CPU resource request of koordlet container | `0` | +| `koordlet.resources.requests.memory` | Memory resource request of koordlet container | `0` | +| `webhookConfiguration.failurePolicy.pods` | The failurePolicy for pods in mutating webhook configuration | `Ignore` | +| `webhookConfiguration.failurePolicy.elasticquotas` | The failurePolicy for elasticQuotas in all webhook configuration | `Ignore` | +| `webhookConfiguration.failurePolicy.nodeStatus` | The failurePolicy for node.status in all webhook configuration | `Ignore` | +| `webhookConfiguration.failurePolicy.nodes` | The failurePolicy for nodes in all webhook configuration | `Ignore` | +| `webhookConfiguration.timeoutSeconds` | The timeoutSeconds for all webhook configuration | `30` | +| `crds.managed` | Koordinator will not install CRDs with chart if this is false | `true` | +| `imagePullSecrets` | The list of image pull secrets for koordinator image | `false` | + +Specify each parameter using the `--set key=value[,key=value]` argument to `helm install` or `helm upgrade`. + +### Optional: feature-gate + +Feature-gate controls some influential features in Koordinator: + +| Name | Description | Default | Effect (if closed) | +| ------------------------- | ---------------------------------------------------------------- | ------- | -------------------------------------- | +| `PodMutatingWebhook` | Whether to open a mutating webhook for Pod **create** | `true` | Don't inject koordinator.sh/qosClass, koordinator.sh/priority and don't replace koordinator extend resources ad so on | +| `PodValidatingWebhook` | Whether to open a validating webhook for Pod **create/update** | `true` | It is possible to create some Pods that do not conform to the Koordinator specification, causing some unpredictable problems | + + +If you want to configure the feature-gate, just set the parameter when install or upgrade. Such as: + +```bash +$ helm install koordinator https://... --set featureGates="PodMutatingWebhook=true\,PodValidatingWebhook=true" +``` + +If you want to enable all feature-gates, set the parameter as `featureGates=AllAlpha=true`. + +### Optional: the local image for China + +If you are in China and have problem to pull image from official DockerHub, you can use the registry hosted on Alibaba Cloud: + +```bash +$ helm install koordinator https://... --set imageRepositoryHost=registry.cn-beijing.aliyuncs.com +``` diff --git a/versions/v1.4.1/templates/_helpers.tpl b/versions/v1.4.1/templates/_helpers.tpl new file mode 100644 index 0000000..56eb792 --- /dev/null +++ b/versions/v1.4.1/templates/_helpers.tpl @@ -0,0 +1,97 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "koordinator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "koordinator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "koordinator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "koordinator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "koordinator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Lookup existing immutatble resources +*/}} +{{- define "webhookServiceSpec" -}} +{{- $service := lookup "v1" "Service" .Values.installation.namespace "koordinator-webhook-service" -}} +{{- if $service -}} +{{ if $service.spec.clusterIP -}} +clusterIP: {{ $service.spec.clusterIP }} +{{- end }} +{{ if $service.spec.clusterIPs -}} +clusterIPs: + {{ $service.spec.clusterIPs }} +{{- end }} +{{ if $service.spec.ipFamilyPolicy -}} +ipFamilyPolicy: {{ $service.spec.ipFamilyPolicy }} +{{- end }} +{{ if $service.spec.ipFamilies -}} +ipFamilies: + {{ $service.spec.ipFamilies }} +{{- end }} +{{ if $service.spec.type -}} +type: {{ $service.spec.type }} +{{- end }} +{{ if $service.spec.ipFamily -}} +ipFamily: {{ $service.spec.ipFamily }} +{{- end }} +{{- end -}} +ports: +- port: 443 + targetPort: {{ .Values.manager.webhook.port }} +selector: + koord-app: koord-manager +{{- end -}} + +{{- define "webhookSecretData" -}} +{{- $secret := lookup "v1" "Secret" .Values.installation.namespace "koordinator-webhook-certs" -}} +{{- if $secret -}} +data: +{{- range $k, $v := $secret.data }} + {{ $k }}: {{ $v }} +{{- end }} +{{- end }} +{{- end -}} + +{{- define "serviceAccountManager" -}} +{{- $sa := lookup "v1" "ServiceAccount" .Values.installation.namespace "koord-manager" -}} +{{- if $sa -}} +secrets: +{{- range $v := $sa.secrets }} +- name: {{ $v.name }} +{{- end }} +{{- end }} +{{- end -}} diff --git a/versions/v1.4.1/templates/crd/config.koordinator.sh_clustercolocationprofiles.yaml b/versions/v1.4.1/templates/crd/config.koordinator.sh_clustercolocationprofiles.yaml new file mode 100644 index 0000000..07655ad --- /dev/null +++ b/versions/v1.4.1/templates/crd/config.koordinator.sh_clustercolocationprofiles.yaml @@ -0,0 +1,212 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: clustercolocationprofiles.config.koordinator.sh +spec: + group: config.koordinator.sh + names: + kind: ClusterColocationProfile + listKind: ClusterColocationProfileList + plural: clustercolocationprofiles + singular: clustercolocationprofile + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ClusterColocationProfile is the Schema for the ClusterColocationProfile + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterColocationProfileSpec is a description of a ClusterColocationProfile. + properties: + annotationKeysMapping: + additionalProperties: + type: string + description: AnnotationKeysMapping describes the annotations that + needs to inject into Pod.Annotations with the same values. It sets + the Pod.Annotations[AnnotationsToAnnotations[k]] = Pod.Annotations[k] + for each key k. + type: object + annotations: + additionalProperties: + type: string + description: Annotations describes the k/v pair that needs to inject + into Pod.Annotations + type: object + koordinatorPriority: + description: KoordinatorPriority defines the Pod sub-priority in Koordinator. + The priority value will be injected into Pod as label koordinator.sh/priority. + Various Koordinator components determine the priority of the Pod + in the Koordinator through KoordinatorPriority and the priority + value in PriorityClassName. The higher the value, the higher the + priority. + format: int32 + type: integer + labelKeysMapping: + additionalProperties: + type: string + description: LabelKeysMapping describes the labels that needs to inject + into Pod.Labels with the same values. It sets the Pod.Labels[LabelsToLabels[k]] + = Pod.Labels[k] for each key k. + type: object + labels: + additionalProperties: + type: string + description: Labels describes the k/v pair that needs to inject into + Pod.Labels + type: object + namespaceSelector: + description: NamespaceSelector decides whether to mutate/validate + Pods if the namespace matches the selector. Default to the empty + LabelSelector, which matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + patch: + description: Patch indicates patching podTemplate that will be injected + to the Pod. + x-kubernetes-preserve-unknown-fields: true + priorityClassName: + description: If specified, the priorityClassName and the priority + value defined in PriorityClass will be injected into the Pod. The + PriorityClassName, priority value in PriorityClassName and KoordinatorPriority + will affect the scheduling, preemption and other behaviors of Koordinator + system. + type: string + probability: + anyOf: + - type: integer + - type: string + description: Probability indicates profile will make effect with a + probability. + x-kubernetes-int-or-string: true + qosClass: + description: QoSClass describes the type of Koordinator QoS that the + Pod is running. The value will be injected into Pod as label koordinator.sh/qosClass. + Options are LSE/LSR/LS/BE/SYSTEM. + enum: + - LSE + - LSR + - LS + - BE + - SYSTEM + type: string + schedulerName: + description: If specified, the pod will be dispatched by specified + scheduler. + type: string + selector: + description: Selector decides whether to mutate/validate Pods if the + Pod matches the selector. Default to the empty LabelSelector, which + matches everything. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + type: object + status: + description: ClusterColocationProfileStatus represents information about + the status of a ClusterColocationProfile. + type: object + type: object + served: true + storage: true + subresources: + status: {} + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/quota.koordinator.sh_elasticquotaprofiles.yaml b/versions/v1.4.1/templates/crd/quota.koordinator.sh_elasticquotaprofiles.yaml new file mode 100644 index 0000000..396300d --- /dev/null +++ b/versions/v1.4.1/templates/crd/quota.koordinator.sh_elasticquotaprofiles.yaml @@ -0,0 +1,106 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: elasticquotaprofiles.quota.koordinator.sh +spec: + group: quota.koordinator.sh + names: + kind: ElasticQuotaProfile + listKind: ElasticQuotaProfileList + plural: elasticquotaprofiles + shortNames: + - eqp + singular: elasticquotaprofile + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + nodeSelector: + description: NodeSelector defines a node selector to select nodes. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the key + and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to + a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + quotaLabels: + additionalProperties: + type: string + description: QuotaLabels defines the labels of the quota. + type: object + quotaName: + description: QuotaName defines the associated quota name of the profile. + type: string + resourceRatio: + description: ResourceRatio is a ratio, we will use it to fix the resource + fragmentation problem. If the total resource is 100 and the resource + ratio is 0.9, the allocable resource is 100*0.9=90 + type: string + required: + - nodeSelector + - quotaName + type: object + status: + type: object + type: object + served: true + storage: true + +{{- end }} \ No newline at end of file diff --git a/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_devices.yaml b/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_devices.yaml new file mode 100644 index 0000000..3a07565 --- /dev/null +++ b/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_devices.yaml @@ -0,0 +1,172 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: devices.scheduling.koordinator.sh +spec: + group: scheduling.koordinator.sh + names: + kind: Device + listKind: DeviceList + plural: devices + singular: device + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device is normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties that can + be used to organize and categorize (scope and select) objects + type: object + minor: + description: Minor represents the Minor number of Device, starting + from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical id of Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource name, quantity) + pairs + type: object + topology: + description: Topology represents the topology information about + the device + properties: + busID: + description: BusID is the domain:bus:device.function formatted + identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node to which the + device belongs, it should be unique across different CPU + Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch to which the + device is connected, it should be unique across difference + NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket to which the + device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual function devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual Function properties + that can be used to organize and categorize (scope and + select) objects + type: object + vfs: + description: VFs are the virtual function devices which + belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE virtual function + device + type: string + minor: + description: Minor represents the Minor number of + VirtualFunction, starting from 0, used to identify + virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + type: object + status: + properties: + allocations: + items: + properties: + entries: + items: + properties: + minors: + items: + format: int32 + type: integer + type: array + name: + type: string + namespace: + type: string + uuid: + type: string + type: object + type: array + type: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_podmigrationjobs.yaml b/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_podmigrationjobs.yaml new file mode 100644 index 0000000..134568f --- /dev/null +++ b/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_podmigrationjobs.yaml @@ -0,0 +1,522 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: podmigrationjobs.scheduling.koordinator.sh +spec: + group: scheduling.koordinator.sh + names: + kind: PodMigrationJob + listKind: PodMigrationJobList + plural: podmigrationjobs + shortNames: + - pmj + singular: podmigrationjob + scope: Cluster + versions: + - additionalPrinterColumns: + - description: The phase of PodMigrationJob + jsonPath: .status.phase + name: Phase + type: string + - description: The status of PodMigrationJob + jsonPath: .status.status + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .spec.reservationOptions.reservationRef.name + name: Reservation + type: string + - jsonPath: .spec.podRef.namespace + name: PodNamespace + type: string + - jsonPath: .spec.podRef.name + name: Pod + type: string + - jsonPath: .status.podRef.name + name: NewPod + type: string + - jsonPath: .spec.ttl + name: TTL + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + deleteOptions: + description: DeleteOptions defines the deleting options for the migrated + Pod and preempted Pods + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + dryRun: + description: 'When present, indicates that modifications should + not be persisted. An invalid or unrecognized dryRun directive + will result in an error response and no further processing of + the request. Valid values are: - All: all dry run stages will + be processed' + items: + type: string + type: array + gracePeriodSeconds: + description: The duration in seconds before the object should + be deleted. Value must be non-negative integer. The value zero + indicates delete immediately. If this value is nil, the default + grace period for the specified type will be used. Defaults to + a per object value if not specified. zero means delete immediately. + format: int64 + type: integer + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + orphanDependents: + description: 'Deprecated: please use the PropagationPolicy, this + field will be deprecated in 1.7. Should the dependent objects + be orphaned. If true/false, the "orphan" finalizer will be added + to/removed from the object''s finalizers list. Either this field + or PropagationPolicy may be set, but not both.' + type: boolean + preconditions: + description: Must be fulfilled before a deletion is carried out. + If not possible, a 409 Conflict status will be returned. + properties: + resourceVersion: + description: Specifies the target ResourceVersion + type: string + uid: + description: Specifies the target UID. + type: string + type: object + propagationPolicy: + description: 'Whether and how garbage collection will be performed. + Either this field or OrphanDependents may be set, but not both. + The default policy is decided by the existing finalizer set + in the metadata.finalizers and the resource-specific default + policy. Acceptable values are: ''Orphan'' - orphan the dependents; + ''Background'' - allow the garbage collector to delete the dependents + in the background; ''Foreground'' - a cascading policy that + deletes all dependents in the foreground.' + type: string + type: object + mode: + description: Mode represents the operating mode of the Job Default + is PodMigrationJobModeReservationFirst + type: string + paused: + description: Paused indicates whether the PodMigrationJob should to + work or not. Default is false + type: boolean + podRef: + description: PodRef represents the Pod that be migrated + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + reservationOptions: + description: ReservationOptions defines the Reservation options for + migrated Pod + properties: + preemptionOptions: + description: PreemptionOption decides whether to preempt other + Pods. The preemption is safe and reserves resources for preempted + Pods. + type: object + reservationRef: + description: ReservationRef if specified, PodMigrationJob will + check if the status of Reservation is available. ReservationRef + if not specified, PodMigrationJob controller will create Reservation + by Template, and update the ReservationRef to reference the + Reservation + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part + of an object. TODO: this design is not final and this field + is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + template: + description: Template is the object that describes the Reservation + that will be created if not specified ReservationRef + x-kubernetes-preserve-unknown-fields: true + type: object + ttl: + description: TTL controls the PodMigrationJob timeout duration. + type: string + required: + - podRef + type: object + status: + properties: + conditions: + description: Conditions records the stats of PodMigrationJob + items: + properties: + lastProbeTime: + description: Last time we probed the condition. + format: date-time + nullable: true + type: string + lastTransitionTime: + description: Last time the condition transitioned from one status + to another. + format: date-time + nullable: true + type: string + message: + description: Human-readable message indicating details about + last transition. + type: string + reason: + description: Unique, one-word, CamelCase reason for the condition's + last transition. + type: string + status: + description: Status is the status of the condition. Can be True, + False, Unknown. + type: string + type: + description: Type is the type of the condition. + type: string + required: + - status + - type + type: object + type: array + message: + description: Message represents a human-readable message indicating + details about why the PodMigrationJob is in this state. + type: string + nodeName: + description: NodeName represents the node's name of migrated Pod + type: string + phase: + description: PodMigrationJobPhase represents the phase of a PodMigrationJob + is a simple, high-level summary of where the PodMigrationJob is + in its lifecycle. e.g. Pending/Running/Failed + type: string + podRef: + description: PodRef represents the newly created Pod after being migrated + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + preemptedPodsRef: + description: PreemptedPodsRef represents the Pods that be preempted + items: + description: "ObjectReference contains enough information to let + you inspect or modify the referred object. --- New uses of this + type are discouraged because of difficulty describing its usage + when embedded in APIs. 1. Ignored fields. It includes many fields + which are not generally honored. For instance, ResourceVersion + and FieldPath are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for individual + usage. In most embedded usages, there are particular restrictions + like, \"must refer only to types A and B\" or \"UID not honored\" + or \"name must be restricted\". Those cannot be well described + when embedded. 3. Inconsistent validation. Because the usages + are different, the validation rules are different by usage, which + makes it hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency is + on the group,resource tuple and the version of the actual struct + is irrelevant. 5. We cannot easily change it. Because this type + is embedded in many locations, updates to this type will affect + numerous schemas. Don't make new APIs embed an underspecified + API type they do not control. \n Instead of using this type, create + a locally provided and used type that is well-focused on your + reference. For example, ServiceReferences for admission registration: + https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + preemptedPodsReservation: + description: PreemptedPodsReservations records information about Reservations + created due to preemption + items: + properties: + name: + description: Name represents the name of Reservation + type: string + namespace: + description: Namespace represents the namespace of Reservation + type: string + nodeName: + description: NodeName represents the assigned node for Reservation + by scheduler + type: string + phase: + description: Phase represents the Phase of Reservation + type: string + podsRef: + description: PodsRef represents the newly created Pods after + being preempted + items: + description: "ObjectReference contains enough information + to let you inspect or modify the referred object. --- New + uses of this type are discouraged because of difficulty + describing its usage when embedded in APIs. 1. Ignored fields. + \ It includes many fields which are not generally honored. + \ For instance, ResourceVersion and FieldPath are both very + rarely valid in actual usage. 2. Invalid usage help. It + is impossible to add specific help for individual usage. + \ In most embedded usages, there are particular restrictions + like, \"must refer only to types A and B\" or \"UID not + honored\" or \"name must be restricted\". Those cannot be + well described when embedded. 3. Inconsistent validation. + \ Because the usages are different, the validation rules + are different by usage, which makes it hard for users to + predict what will happen. 4. The fields are both imprecise + and overly precise. Kind is not a precise mapping to a + URL. This can produce ambiguity during interpretation and + require a REST mapping. In most cases, the dependency is + on the group,resource tuple and the version of the actual + struct is irrelevant. 5. We cannot easily change it. Because + this type is embedded in many locations, updates to this + type will affect numerous schemas. Don't make new APIs + embed an underspecified API type they do not control. \n + Instead of using this type, create a locally provided and + used type that is well-focused on your reference. For example, + ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this + pod). This syntax is chosen only to have some well-defined + way of referencing a part of an object. TODO: this design + is not final and this field is subject to change in + the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + preemptedPodRef: + description: PreemptedPodRef represents the Pod that be preempted + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: object + type: array + reason: + description: Reason represents a brief CamelCase message indicating + details about why the PodMigrationJob is in this state. + type: string + status: + description: Status represents the current status of PodMigrationJob + e.g. ReservationCreated + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_reservations.yaml b/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_reservations.yaml new file mode 100644 index 0000000..40f0d9a --- /dev/null +++ b/versions/v1.4.1/templates/crd/scheduling.koordinator.sh_reservations.yaml @@ -0,0 +1,356 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: reservations.scheduling.koordinator.sh +spec: + group: scheduling.koordinator.sh + names: + kind: Reservation + listKind: ReservationList + plural: reservations + singular: reservation + scope: Cluster + versions: + - additionalPrinterColumns: + - description: The phase of reservation + jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .spec.ttl + name: TTL + type: string + - jsonPath: .spec.expires + name: Expires + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Reservation is the Schema for the reservation API. A Reservation + object is non-namespaced. Any namespaced affinity/anti-affinity of reservation + scheduling can be specified in the spec.template. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + allocateOnce: + default: true + description: When `AllocateOnce` is set, the reserved resources are + only available for the first owner who allocates successfully and + are not allocatable to other owners anymore. Defaults to true. + type: boolean + allocatePolicy: + description: AllocatePolicy represents the allocation policy of reserved + resources that Reservation expects. + enum: + - Aligned + - Restricted + type: string + expires: + description: Expired timestamp when the reservation is expected to + expire. If both `expires` and `ttl` are set, `expires` is checked + first. `expires` and `ttl` are mutually exclusive. Defaults to being + set dynamically at runtime based on the `ttl`. + format: date-time + type: string + owners: + description: Specify the owners who can allocate the reserved resources. + Multiple owner selectors and ORed. + items: + description: ReservationOwner indicates the owner specification + which can allocate reserved resources. + minProperties: 1 + properties: + controller: + properties: + apiVersion: + description: API version of the referent. + type: string + blockOwnerDeletion: + description: If true, AND if the owner has the "foregroundDeletion" + finalizer, then the owner cannot be deleted from the key-value + store until this reference is removed. See https://kubernetes.io/docs/concepts/architecture/garbage-collection/#foreground-deletion + for how the garbage collector interacts with this field + and enforces the foreground deletion. Defaults to false. + To set this field, a user needs "delete" permission of + the owner, otherwise 422 (Unprocessable Entity) will be + returned. + type: boolean + controller: + description: If true, this reference points to the managing + controller. + type: boolean + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + type: string + uid: + description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + labelSelector: + description: A label selector is a label query over a set of + resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. A + null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + object: + description: Multiple field selectors are ANDed. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead + of an entire object, this string should contain a valid + JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container + within a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that + triggered the event) or if no container name is specified + "spec.containers[2]" (container with index 2 in this pod). + This syntax is chosen only to have some well-defined way + of referencing a part of an object. TODO: this design + is not final and this field is subject to change in the + future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: object + minItems: 1 + type: array + preAllocation: + description: By default, the resources requirements of reservation + (specified in `template.spec`) is filtered by whether the node has + sufficient free resources (i.e. Reservation Request < Node Free). + When `preAllocation` is set, the scheduler will skip this validation + and allow overcommitment. The scheduled reservation would be waiting + to be available until free resources are sufficient. + type: boolean + template: + description: Template defines the scheduling requirements (resources, + affinities, images, ...) processed by the scheduler just like a + normal pod. If the `template.spec.nodeName` is specified, the scheduler + will not choose another node but reserve resources on the specified + node. + x-kubernetes-preserve-unknown-fields: true + ttl: + default: 24h + description: Time-to-Live period for the reservation. `expires` and + `ttl` are mutually exclusive. Defaults to 24h. Set 0 to disable + expiration. + type: string + unschedulable: + description: Unschedulable controls reservation schedulability of + new pods. By default, reservation is schedulable. + type: boolean + required: + - owners + - template + type: object + status: + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resource reserved and allocatable for owners. + type: object + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resource allocated by current owners. + type: object + conditions: + description: The `conditions` indicate the messages of reason why + the reservation is still pending. + items: + properties: + lastProbeTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + currentOwners: + description: Current resource owners which allocated the reservation + resources. + items: + description: "ObjectReference contains enough information to let + you inspect or modify the referred object. --- New uses of this + type are discouraged because of difficulty describing its usage + when embedded in APIs. 1. Ignored fields. It includes many fields + which are not generally honored. For instance, ResourceVersion + and FieldPath are both very rarely valid in actual usage. 2. Invalid + usage help. It is impossible to add specific help for individual + usage. In most embedded usages, there are particular restrictions + like, \"must refer only to types A and B\" or \"UID not honored\" + or \"name must be restricted\". Those cannot be well described + when embedded. 3. Inconsistent validation. Because the usages + are different, the validation rules are different by usage, which + makes it hard for users to predict what will happen. 4. The fields + are both imprecise and overly precise. Kind is not a precise + mapping to a URL. This can produce ambiguity during interpretation + and require a REST mapping. In most cases, the dependency is + on the group,resource tuple and the version of the actual struct + is irrelevant. 5. We cannot easily change it. Because this type + is embedded in many locations, updates to this type will affect + numerous schemas. Don't make new APIs embed an underspecified + API type they do not control. \n Instead of using this type, create + a locally provided and used type that is well-focused on your + reference. For example, ServiceReferences for admission registration: + https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 + ." + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + type: array + nodeName: + description: Name of node the reservation is scheduled on. + type: string + phase: + description: The `phase` indicates whether is reservation is waiting + for process, available to allocate or failed/expired to get cleanup. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_elasticquotas.yaml b/versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_elasticquotas.yaml new file mode 100644 index 0000000..121a856 --- /dev/null +++ b/versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_elasticquotas.yaml @@ -0,0 +1,89 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/52 # edited manually + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: elasticquotas.scheduling.sigs.k8s.io +spec: + group: scheduling.sigs.k8s.io + names: + kind: ElasticQuota + listKind: ElasticQuotaList + plural: elasticquotas + singular: elasticquota + shortNames: # edited manually + - eq # edited manually + - eqs # edited manually + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: ElasticQuota sets elastic quota restrictions per namespace + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ElasticQuotaSpec defines the Min and Max for Quota. + properties: + max: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Max is the set of desired max limits for each named resource. + The usage of max is based on the resource configurations of successfully + scheduled pods. + type: object + min: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Min is the set of desired guaranteed limits for each + named resource. + type: object + type: object + status: + description: ElasticQuotaStatus defines the observed use. + properties: + used: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Used is the current observed total usage of the resource + in the namespace. + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_podgroups.yaml b/versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_podgroups.yaml new file mode 100644 index 0000000..d5874f4 --- /dev/null +++ b/versions/v1.4.1/templates/crd/scheduling.sigs.k8s.io_podgroups.yaml @@ -0,0 +1,108 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/50 # edited manually + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: podgroups.scheduling.sigs.k8s.io +spec: + group: scheduling.sigs.k8s.io + names: + kind: PodGroup + listKind: PodGroupList + plural: podgroups + shortNames: + - pg + - pgs + singular: podgroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: PodGroup is a collection of Pod; used for batch workload. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Specification of the desired behavior of the pod group. + properties: + minMember: + description: MinMember defines the minimal number of members/tasks + to run the pod group; if there's not enough resources to start all + tasks, the scheduler will not start anyone. + format: int32 + type: integer + minResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: MinResources defines the minimal resource of members/tasks + to run the pod group; if there's not enough resources to start all + tasks, the scheduler will not start anyone. + type: object + scheduleTimeoutSeconds: + description: ScheduleTimeoutSeconds defines the maximal time of members/tasks + to wait before run the pod group; + format: int32 + type: integer + type: object + status: + description: Status represents the current information about a pod group. + This data may not be up to date. + properties: + failed: + description: The number of pods which reached phase Failed. + format: int32 + type: integer + occupiedBy: + description: OccupiedBy marks the workload (e.g., deployment, statefulset) + UID that occupy the podgroup. It is empty if not initialized. + type: string + phase: + description: Current phase of PodGroup. + type: string + running: + description: The number of actively running pods. + format: int32 + type: integer + scheduleStartTime: + description: ScheduleStartTime of the group + format: date-time + type: string + scheduled: + description: The number of actively running pods. + format: int32 + type: integer + succeeded: + description: The number of pods which reached phase Succeeded. + format: int32 + type: integer + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/slo.koordinator.sh_nodemetrics.yaml b/versions/v1.4.1/templates/crd/slo.koordinator.sh_nodemetrics.yaml new file mode 100644 index 0000000..2d826c4 --- /dev/null +++ b/versions/v1.4.1/templates/crd/slo.koordinator.sh_nodemetrics.yaml @@ -0,0 +1,1009 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: nodemetrics.slo.koordinator.sh +spec: + group: slo.koordinator.sh + names: + kind: NodeMetric + listKind: NodeMetricList + plural: nodemetrics + singular: nodemetric + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeMetric is the Schema for the nodemetrics API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeMetricSpec defines the desired state of NodeMetric + properties: + metricCollectPolicy: + description: CollectPolicy defines the Metric collection policy + properties: + aggregateDurationSeconds: + description: AggregateDurationSeconds represents the aggregation + period in seconds + format: int64 + type: integer + nodeAggregatePolicy: + description: NodeAggregatePolicy represents the target grain of + node aggregated usage + properties: + durations: + items: + type: string + type: array + type: object + nodeMemoryCollectPolicy: + description: NodeMemoryPolicy represents apply which method collect + memory info + enum: + - usageWithHotPageCache + - usageWithoutPageCache + - usageWithPageCache + type: string + reportIntervalSeconds: + description: ReportIntervalSeconds represents the report period + in seconds + format: int64 + type: integer + type: object + type: object + status: + description: NodeMetricStatus defines the observed state of NodeMetric + properties: + hostApplicationMetric: + description: HostApplicationMetric contains the metrics of out-out-band + applications on node. + items: + properties: + name: + description: Name of the host application + type: string + priority: + description: Priority class of the application + type: string + qos: + description: QoS class of the application + type: string + usage: + description: Resource usage of the host application + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device is + normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize (scope + and select) objects + type: object + minor: + description: Minor represents the Minor number of + Device, starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical id of + Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource name, + quantity) pairs + type: object + topology: + description: Topology represents the topology information + about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node to + which the device belongs, it should be unique + across different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch to + which the device is connected, it should be + unique across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket + to which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual function + devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual Function + properties that can be used to organize and + categorize (scope and select) objects + type: object + vfs: + description: VFs are the virtual function devices + which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE virtual + function device + type: string + minor: + description: Minor represents the Minor + number of VirtualFunction, starting + from 0, used to identify virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + type: object + type: object + type: array + nodeMetric: + description: NodeMetric contains the metrics for this node. + properties: + aggregatedNodeUsages: + description: AggregatedNodeUsages will report only if there are + enough samples + items: + properties: + duration: + type: string + usage: + additionalProperties: + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device + is normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize + (scope and select) objects + type: object + minor: + description: Minor represents the Minor number + of Device, starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical + id of Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource + name, quantity) pairs + type: object + topology: + description: Topology represents the topology + information about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node + to which the device belongs, it should + be unique across different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch + to which the device is connected, it should + be unique across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket + to which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual + function devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual + Function properties that can be used + to organize and categorize (scope and + select) objects + type: object + vfs: + description: VFs are the virtual function + devices which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE + virtual function device + type: string + minor: + description: Minor represents the + Minor number of VirtualFunction, + starting from 0, used to identify + virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, + quantity) pairs. + type: object + type: object + type: object + type: object + type: array + aggregatedSystemUsages: + description: AggregatedSystemUsages will report only if there + are enough samples Deleted pods will be excluded during aggregation + items: + properties: + duration: + type: string + usage: + additionalProperties: + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device + is normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize + (scope and select) objects + type: object + minor: + description: Minor represents the Minor number + of Device, starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical + id of Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource + name, quantity) pairs + type: object + topology: + description: Topology represents the topology + information about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node + to which the device belongs, it should + be unique across different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch + to which the device is connected, it should + be unique across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket + to which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual + function devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual + Function properties that can be used + to organize and categorize (scope and + select) objects + type: object + vfs: + description: VFs are the virtual function + devices which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE + virtual function device + type: string + minor: + description: Minor represents the + Minor number of VirtualFunction, + starting from 0, used to identify + virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, + quantity) pairs. + type: object + type: object + type: object + type: object + type: array + nodeUsage: + description: NodeUsage is the total resource usage of node + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device is + normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize (scope + and select) objects + type: object + minor: + description: Minor represents the Minor number of Device, + starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical id of + Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource name, quantity) + pairs + type: object + topology: + description: Topology represents the topology information + about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node to which + the device belongs, it should be unique across + different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch to + which the device is connected, it should be unique + across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket to + which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual function + devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual Function + properties that can be used to organize and + categorize (scope and select) objects + type: object + vfs: + description: VFs are the virtual function devices + which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE virtual + function device + type: string + minor: + description: Minor represents the Minor + number of VirtualFunction, starting from + 0, used to identify virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + type: object + systemUsage: + description: SystemUsage is the resource usage of daemon processes + and OS kernel, calculated by `NodeUsage - sum(podUsage)` + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device is + normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize (scope + and select) objects + type: object + minor: + description: Minor represents the Minor number of Device, + starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical id of + Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource name, quantity) + pairs + type: object + topology: + description: Topology represents the topology information + about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node to which + the device belongs, it should be unique across + different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch to + which the device is connected, it should be unique + across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket to + which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual function + devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual Function + properties that can be used to organize and + categorize (scope and select) objects + type: object + vfs: + description: VFs are the virtual function devices + which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE virtual + function device + type: string + minor: + description: Minor represents the Minor + number of VirtualFunction, starting from + 0, used to identify virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + type: object + type: object + podsMetric: + description: PodsMetric contains the metrics for pods belong to this + node. + items: + properties: + extensions: + description: Third party extensions for PodMetric + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + namespace: + type: string + podUsage: + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device is + normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize (scope + and select) objects + type: object + minor: + description: Minor represents the Minor number of + Device, starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical id of + Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource name, + quantity) pairs + type: object + topology: + description: Topology represents the topology information + about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node to + which the device belongs, it should be unique + across different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch to + which the device is connected, it should be + unique across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket + to which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual function + devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual Function + properties that can be used to organize and + categorize (scope and select) objects + type: object + vfs: + description: VFs are the virtual function devices + which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE virtual + function device + type: string + minor: + description: Minor represents the Minor + number of VirtualFunction, starting + from 0, used to identify virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + type: object + priority: + description: Priority class of the application + type: string + qos: + description: QoS class of the application + type: string + type: object + type: array + prodReclaimableMetric: + description: ProdReclaimableMetric is the indicator statistics of + Prod type resources reclaimable + properties: + resource: + description: Resource is the resource usage of the prediction + properties: + devices: + items: + properties: + health: + default: false + description: Health indicates whether the device is + normal + type: boolean + id: + description: UUID represents the UUID of device + type: string + labels: + additionalProperties: + type: string + description: Labels represents the device properties + that can be used to organize and categorize (scope + and select) objects + type: object + minor: + description: Minor represents the Minor number of Device, + starting from 0 + format: int32 + type: integer + moduleID: + description: ModuleID represents the physical id of + Device + format: int32 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is a set of (resource name, quantity) + pairs + type: object + topology: + description: Topology represents the topology information + about the device + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE device + type: string + nodeID: + description: NodeID is the ID of NUMA Node to which + the device belongs, it should be unique across + different CPU Sockets + format: int32 + type: integer + pcieID: + description: PCIEID is the ID of PCIE Switch to + which the device is connected, it should be unique + across difference NUMANodes + type: string + socketID: + description: SocketID is the ID of CPU Socket to + which the device belongs + format: int32 + type: integer + required: + - nodeID + - pcieID + - socketID + type: object + type: + description: Type represents the type of device + type: string + vfGroups: + description: VFGroups represents the virtual function + devices + items: + properties: + labels: + additionalProperties: + type: string + description: Labels represents the Virtual Function + properties that can be used to organize and + categorize (scope and select) objects + type: object + vfs: + description: VFs are the virtual function devices + which belong to the group + items: + properties: + busID: + description: BusID is the domain:bus:device.function + formatted identifier of PCI/PCIE virtual + function device + type: string + minor: + description: Minor represents the Minor + number of VirtualFunction, starting from + 0, used to identify virtual function. + format: int32 + type: integer + required: + - minor + type: object + type: array + type: object + type: array + required: + - health + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: ResourceList is a set of (resource name, quantity) + pairs. + type: object + type: object + type: object + updateTime: + description: UpdateTime is the last time this NodeMetric was updated. + format: date-time + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/slo.koordinator.sh_nodeslos.yaml b/versions/v1.4.1/templates/crd/slo.koordinator.sh_nodeslos.yaml new file mode 100644 index 0000000..4d4c8fd --- /dev/null +++ b/versions/v1.4.1/templates/crd/slo.koordinator.sh_nodeslos.yaml @@ -0,0 +1,1601 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: nodeslos.slo.koordinator.sh +spec: + group: slo.koordinator.sh + names: + kind: NodeSLO + listKind: NodeSLOList + plural: nodeslos + singular: nodeslo + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeSLO is the Schema for the nodeslos API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeSLOSpec defines the desired state of NodeSLO + properties: + cpuBurstStrategy: + description: CPU Burst Strategy + properties: + cfsQuotaBurstPercent: + description: pod cfs quota scale up ceil percentage, default = + 300 (300%) + format: int64 + type: integer + cfsQuotaBurstPeriodSeconds: + description: specifies a period of time for pod can use at burst, + default = -1 (unlimited) + format: int64 + type: integer + cpuBurstPercent: + description: 'cpu burst percentage for setting cpu.cfs_burst_us, + legal range: [0, 10000], default as 1000 (1000%)' + format: int64 + maximum: 10000 + minimum: 0 + type: integer + policy: + type: string + sharePoolThresholdPercent: + description: scale down cfs quota if node cpu overload, default + = 50 + format: int64 + type: integer + type: object + extensions: + description: Third party extensions for NodeSLO + type: object + x-kubernetes-preserve-unknown-fields: true + hostApplications: + description: QoS management for out-of-band applications + items: + description: HostApplicationSpec describes the QoS management for + out-out-band applications on node + properties: + cgroupPath: + description: Optional, defines the host cgroup configuration, + use default if not specified according to priority and qos + properties: + base: + description: cgroup base dir, the format is various across + cgroup drivers + type: string + parentDir: + description: cgroup parent path under base dir + type: string + relativePath: + description: cgroup relative path under parent dir + type: string + type: object + name: + type: string + priority: + description: Priority class of the application + type: string + qos: + description: QoS class of the application + type: string + strategy: + description: QoS Strategy of host application + type: object + type: object + type: array + resourceQOSStrategy: + description: QoS config strategy for pods of different qos-class + properties: + beClass: + description: ResourceQOS for BE pods. + properties: + blkioQOS: + properties: + blocks: + items: + properties: + ioCfg: + properties: + ioWeightPercent: + description: 'This field is used to set the + weight of a sub-group. Default value: 100. + Valid values: 1 to 100.' + format: int64 + maximum: 100 + minimum: 1 + type: integer + readBPS: + description: Throttling of throughput The value + is set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readIOPS: + description: Throttling of IOPS The value is + set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readLatency: + description: 'Configure the weight-based throttling + feature of blk-iocost Only used for RootClass + After blk-iocost is enabled, the kernel calculates + the proportion of requests that exceed the + read or write latency threshold out of all + requests. When the proportion is greater than + the read or write latency percentile (95%), + the kernel considers the disk to be saturated + and reduces the rate at which requests are + sent to the disk. the read latency threshold. + Unit: microseconds.' + format: int64 + type: integer + writeBPS: + format: int64 + minimum: 0 + type: integer + writeIOPS: + format: int64 + minimum: 0 + type: integer + writeLatency: + description: 'the write latency threshold. Unit: + microseconds.' + format: int64 + type: integer + type: object + name: + type: string + type: + type: string + type: object + type: array + enable: + type: boolean + type: object + cpuQOS: + description: CPUQOSCfg stores node-level config of cpu qos + properties: + coreExpeller: + description: 'whether pods of the QoS class can expel + the cgroup idle pods at the SMT-level. default = false + If set to true, pods of this QoS will use a dedicated + core sched group for noise clean with the SchedIdle + pods. NOTE: It takes effect if cpuPolicy = "coreSched".' + type: boolean + enable: + description: Enable indicates whether the cpu qos is enabled. + type: boolean + groupIdentity: + description: 'group identity value for pods, default = + 0 NOTE: It takes effect if cpuPolicy = "groupIdentity".' + format: int64 + type: integer + schedIdle: + description: 'cpu.idle value for pods, default = 0. `1` + means using SCHED_IDLE. CGroup Idle (introduced since + mainline Linux 5.15): https://lore.kernel.org/lkml/162971078674.25758.15464079371945307825.tip-bot2@tip-bot2/#r + NOTE: It takes effect if cpuPolicy = "coreSched".' + format: int64 + type: integer + type: object + memoryQOS: + description: MemoryQOSCfg stores node-level config of memory + qos + properties: + enable: + description: 'Enable indicates whether the memory qos + is enabled (default: false). This field is used for + node-level control, while pod-level configuration is + done with MemoryQOS and `Policy` instead of an `Enable` + option. Please view the differences between MemoryQOSCfg + and PodMemoryQOSConfig structs.' + type: boolean + lowLimitPercent: + description: 'LowLimitPercent specifies the lowLimitFactor + percentage to calculate `memory.low`, which TRIES BEST + protecting memory from global reclamation when memory + usage does not exceed the low limit unless no unprotected + memcg can be reclaimed. NOTE: `memory.low` should be + larger than `memory.min`. If spec.requests.memory == + spec.limits.memory, pod `memory.low` and `memory.high` + become invalid, while `memory.wmark_ratio` is still + in effect. Close: 0.' + format: int64 + minimum: 0 + type: integer + minLimitPercent: + description: 'memcg qos If enabled, memcg qos will be + set by the agent, where some fields are implicitly calculated + from pod spec. 1. `memory.min` := spec.requests.memory + * minLimitFactor / 100 (use 0 if requests.memory is + not set) 2. `memory.low` := spec.requests.memory * lowLimitFactor + / 100 (use 0 if requests.memory is not set) 3. `memory.limit_in_bytes` + := spec.limits.memory (set $node.allocatable.memory + if limits.memory is not set) 4. `memory.high` := floor[(spec.requests.memory + + throttlingFactor / 100 * (memory.limit_in_bytes or + node allocatable memory - spec.requests.memory))/pageSize] + * pageSize MinLimitPercent specifies the minLimitFactor + percentage to calculate `memory.min`, which protects + memory from global reclamation when memory usage does + not exceed the min limit. Close: 0.' + format: int64 + minimum: 0 + type: integer + oomKillGroup: + format: int64 + type: integer + priority: + format: int64 + type: integer + priorityEnable: + description: 'TODO: enhance the usages of oom priority + and oom kill group' + format: int64 + type: integer + throttlingPercent: + description: 'ThrottlingPercent specifies the throttlingFactor + percentage to calculate `memory.high` with pod memory.limits + or node allocatable memory, which triggers memcg direct + reclamation when memory usage exceeds. Lower the factor + brings more heavier reclaim pressure. Close: 0.' + format: int64 + minimum: 0 + type: integer + wmarkMinAdj: + description: 'wmark_min_adj (Anolis OS required) WmarkMinAdj + specifies `memory.wmark_min_adj` which adjusts per-memcg + threshold for global memory reclamation. Lower the factor + brings later reclamation. The adjustment uses different + formula for different value range. [-25, 0)：global_wmark_min'' + = global_wmark_min + (global_wmark_min - 0) * wmarkMinAdj + (0, 50]：global_wmark_min'' = global_wmark_min + (global_wmark_low + - global_wmark_min) * wmarkMinAdj Close: [LSR:0, LS:0, + BE:0]. Recommended: [LSR:-25, LS:-25, BE:50].' + format: int64 + maximum: 50 + minimum: -25 + type: integer + wmarkRatio: + description: 'wmark_ratio (Anolis OS required) Async memory + reclamation is triggered when cgroup memory usage exceeds + `memory.wmark_high` and the reclamation stops when usage + is below `memory.wmark_low`. Basically, `memory.wmark_high` + := min(memory.high, memory.limit_in_bytes) * memory.memory.wmark_ratio + `memory.wmark_low` := min(memory.high, memory.limit_in_bytes) + * (memory.wmark_ratio - memory.wmark_scale_factor) WmarkRatio + specifies `memory.wmark_ratio` that help calculate `memory.wmark_high`, + which triggers async memory reclamation when memory + usage exceeds. Close: 0. Recommended: 95.' + format: int64 + maximum: 100 + minimum: 0 + type: integer + wmarkScalePermill: + description: 'WmarkScalePermill specifies `memory.wmark_scale_factor` + that helps calculate `memory.wmark_low`, which stops + async memory reclamation when memory usage belows. Close: + 50. Recommended: 20.' + format: int64 + maximum: 1000 + minimum: 1 + type: integer + type: object + networkQOS: + properties: + egressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'EgressLimit describes the maximum network + bandwidth can be used in the egress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + egressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'EgressRequest describes the minimum network + bandwidth guaranteed in the egress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + enable: + type: boolean + ingressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'IngressLimit describes the maximum network + bandwidth can be used in the ingress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + ingressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'IngressRequest describes the minimum network + bandwidth guaranteed in the ingress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + type: object + resctrlQOS: + description: ResctrlQOSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object + type: object + cgroupRoot: + description: ResourceQOS for root cgroup. + properties: + blkioQOS: + properties: + blocks: + items: + properties: + ioCfg: + properties: + ioWeightPercent: + description: 'This field is used to set the + weight of a sub-group. Default value: 100. + Valid values: 1 to 100.' + format: int64 + maximum: 100 + minimum: 1 + type: integer + readBPS: + description: Throttling of throughput The value + is set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readIOPS: + description: Throttling of IOPS The value is + set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readLatency: + description: 'Configure the weight-based throttling + feature of blk-iocost Only used for RootClass + After blk-iocost is enabled, the kernel calculates + the proportion of requests that exceed the + read or write latency threshold out of all + requests. When the proportion is greater than + the read or write latency percentile (95%), + the kernel considers the disk to be saturated + and reduces the rate at which requests are + sent to the disk. the read latency threshold. + Unit: microseconds.' + format: int64 + type: integer + writeBPS: + format: int64 + minimum: 0 + type: integer + writeIOPS: + format: int64 + minimum: 0 + type: integer + writeLatency: + description: 'the write latency threshold. Unit: + microseconds.' + format: int64 + type: integer + type: object + name: + type: string + type: + type: string + type: object + type: array + enable: + type: boolean + type: object + cpuQOS: + description: CPUQOSCfg stores node-level config of cpu qos + properties: + coreExpeller: + description: 'whether pods of the QoS class can expel + the cgroup idle pods at the SMT-level. default = false + If set to true, pods of this QoS will use a dedicated + core sched group for noise clean with the SchedIdle + pods. NOTE: It takes effect if cpuPolicy = "coreSched".' + type: boolean + enable: + description: Enable indicates whether the cpu qos is enabled. + type: boolean + groupIdentity: + description: 'group identity value for pods, default = + 0 NOTE: It takes effect if cpuPolicy = "groupIdentity".' + format: int64 + type: integer + schedIdle: + description: 'cpu.idle value for pods, default = 0. `1` + means using SCHED_IDLE. CGroup Idle (introduced since + mainline Linux 5.15): https://lore.kernel.org/lkml/162971078674.25758.15464079371945307825.tip-bot2@tip-bot2/#r + NOTE: It takes effect if cpuPolicy = "coreSched".' + format: int64 + type: integer + type: object + memoryQOS: + description: MemoryQOSCfg stores node-level config of memory + qos + properties: + enable: + description: 'Enable indicates whether the memory qos + is enabled (default: false). This field is used for + node-level control, while pod-level configuration is + done with MemoryQOS and `Policy` instead of an `Enable` + option. Please view the differences between MemoryQOSCfg + and PodMemoryQOSConfig structs.' + type: boolean + lowLimitPercent: + description: 'LowLimitPercent specifies the lowLimitFactor + percentage to calculate `memory.low`, which TRIES BEST + protecting memory from global reclamation when memory + usage does not exceed the low limit unless no unprotected + memcg can be reclaimed. NOTE: `memory.low` should be + larger than `memory.min`. If spec.requests.memory == + spec.limits.memory, pod `memory.low` and `memory.high` + become invalid, while `memory.wmark_ratio` is still + in effect. Close: 0.' + format: int64 + minimum: 0 + type: integer + minLimitPercent: + description: 'memcg qos If enabled, memcg qos will be + set by the agent, where some fields are implicitly calculated + from pod spec. 1. `memory.min` := spec.requests.memory + * minLimitFactor / 100 (use 0 if requests.memory is + not set) 2. `memory.low` := spec.requests.memory * lowLimitFactor + / 100 (use 0 if requests.memory is not set) 3. `memory.limit_in_bytes` + := spec.limits.memory (set $node.allocatable.memory + if limits.memory is not set) 4. `memory.high` := floor[(spec.requests.memory + + throttlingFactor / 100 * (memory.limit_in_bytes or + node allocatable memory - spec.requests.memory))/pageSize] + * pageSize MinLimitPercent specifies the minLimitFactor + percentage to calculate `memory.min`, which protects + memory from global reclamation when memory usage does + not exceed the min limit. Close: 0.' + format: int64 + minimum: 0 + type: integer + oomKillGroup: + format: int64 + type: integer + priority: + format: int64 + type: integer + priorityEnable: + description: 'TODO: enhance the usages of oom priority + and oom kill group' + format: int64 + type: integer + throttlingPercent: + description: 'ThrottlingPercent specifies the throttlingFactor + percentage to calculate `memory.high` with pod memory.limits + or node allocatable memory, which triggers memcg direct + reclamation when memory usage exceeds. Lower the factor + brings more heavier reclaim pressure. Close: 0.' + format: int64 + minimum: 0 + type: integer + wmarkMinAdj: + description: 'wmark_min_adj (Anolis OS required) WmarkMinAdj + specifies `memory.wmark_min_adj` which adjusts per-memcg + threshold for global memory reclamation. Lower the factor + brings later reclamation. The adjustment uses different + formula for different value range. [-25, 0)：global_wmark_min'' + = global_wmark_min + (global_wmark_min - 0) * wmarkMinAdj + (0, 50]：global_wmark_min'' = global_wmark_min + (global_wmark_low + - global_wmark_min) * wmarkMinAdj Close: [LSR:0, LS:0, + BE:0]. Recommended: [LSR:-25, LS:-25, BE:50].' + format: int64 + maximum: 50 + minimum: -25 + type: integer + wmarkRatio: + description: 'wmark_ratio (Anolis OS required) Async memory + reclamation is triggered when cgroup memory usage exceeds + `memory.wmark_high` and the reclamation stops when usage + is below `memory.wmark_low`. Basically, `memory.wmark_high` + := min(memory.high, memory.limit_in_bytes) * memory.memory.wmark_ratio + `memory.wmark_low` := min(memory.high, memory.limit_in_bytes) + * (memory.wmark_ratio - memory.wmark_scale_factor) WmarkRatio + specifies `memory.wmark_ratio` that help calculate `memory.wmark_high`, + which triggers async memory reclamation when memory + usage exceeds. Close: 0. Recommended: 95.' + format: int64 + maximum: 100 + minimum: 0 + type: integer + wmarkScalePermill: + description: 'WmarkScalePermill specifies `memory.wmark_scale_factor` + that helps calculate `memory.wmark_low`, which stops + async memory reclamation when memory usage belows. Close: + 50. Recommended: 20.' + format: int64 + maximum: 1000 + minimum: 1 + type: integer + type: object + networkQOS: + properties: + egressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'EgressLimit describes the maximum network + bandwidth can be used in the egress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + egressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'EgressRequest describes the minimum network + bandwidth guaranteed in the egress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + enable: + type: boolean + ingressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'IngressLimit describes the maximum network + bandwidth can be used in the ingress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + ingressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'IngressRequest describes the minimum network + bandwidth guaranteed in the ingress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + type: object + resctrlQOS: + description: ResctrlQOSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object + type: object + lsClass: + description: ResourceQOS for LS pods. + properties: + blkioQOS: + properties: + blocks: + items: + properties: + ioCfg: + properties: + ioWeightPercent: + description: 'This field is used to set the + weight of a sub-group. Default value: 100. + Valid values: 1 to 100.' + format: int64 + maximum: 100 + minimum: 1 + type: integer + readBPS: + description: Throttling of throughput The value + is set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readIOPS: + description: Throttling of IOPS The value is + set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readLatency: + description: 'Configure the weight-based throttling + feature of blk-iocost Only used for RootClass + After blk-iocost is enabled, the kernel calculates + the proportion of requests that exceed the + read or write latency threshold out of all + requests. When the proportion is greater than + the read or write latency percentile (95%), + the kernel considers the disk to be saturated + and reduces the rate at which requests are + sent to the disk. the read latency threshold. + Unit: microseconds.' + format: int64 + type: integer + writeBPS: + format: int64 + minimum: 0 + type: integer + writeIOPS: + format: int64 + minimum: 0 + type: integer + writeLatency: + description: 'the write latency threshold. Unit: + microseconds.' + format: int64 + type: integer + type: object + name: + type: string + type: + type: string + type: object + type: array + enable: + type: boolean + type: object + cpuQOS: + description: CPUQOSCfg stores node-level config of cpu qos + properties: + coreExpeller: + description: 'whether pods of the QoS class can expel + the cgroup idle pods at the SMT-level. default = false + If set to true, pods of this QoS will use a dedicated + core sched group for noise clean with the SchedIdle + pods. NOTE: It takes effect if cpuPolicy = "coreSched".' + type: boolean + enable: + description: Enable indicates whether the cpu qos is enabled. + type: boolean + groupIdentity: + description: 'group identity value for pods, default = + 0 NOTE: It takes effect if cpuPolicy = "groupIdentity".' + format: int64 + type: integer + schedIdle: + description: 'cpu.idle value for pods, default = 0. `1` + means using SCHED_IDLE. CGroup Idle (introduced since + mainline Linux 5.15): https://lore.kernel.org/lkml/162971078674.25758.15464079371945307825.tip-bot2@tip-bot2/#r + NOTE: It takes effect if cpuPolicy = "coreSched".' + format: int64 + type: integer + type: object + memoryQOS: + description: MemoryQOSCfg stores node-level config of memory + qos + properties: + enable: + description: 'Enable indicates whether the memory qos + is enabled (default: false). This field is used for + node-level control, while pod-level configuration is + done with MemoryQOS and `Policy` instead of an `Enable` + option. Please view the differences between MemoryQOSCfg + and PodMemoryQOSConfig structs.' + type: boolean + lowLimitPercent: + description: 'LowLimitPercent specifies the lowLimitFactor + percentage to calculate `memory.low`, which TRIES BEST + protecting memory from global reclamation when memory + usage does not exceed the low limit unless no unprotected + memcg can be reclaimed. NOTE: `memory.low` should be + larger than `memory.min`. If spec.requests.memory == + spec.limits.memory, pod `memory.low` and `memory.high` + become invalid, while `memory.wmark_ratio` is still + in effect. Close: 0.' + format: int64 + minimum: 0 + type: integer + minLimitPercent: + description: 'memcg qos If enabled, memcg qos will be + set by the agent, where some fields are implicitly calculated + from pod spec. 1. `memory.min` := spec.requests.memory + * minLimitFactor / 100 (use 0 if requests.memory is + not set) 2. `memory.low` := spec.requests.memory * lowLimitFactor + / 100 (use 0 if requests.memory is not set) 3. `memory.limit_in_bytes` + := spec.limits.memory (set $node.allocatable.memory + if limits.memory is not set) 4. `memory.high` := floor[(spec.requests.memory + + throttlingFactor / 100 * (memory.limit_in_bytes or + node allocatable memory - spec.requests.memory))/pageSize] + * pageSize MinLimitPercent specifies the minLimitFactor + percentage to calculate `memory.min`, which protects + memory from global reclamation when memory usage does + not exceed the min limit. Close: 0.' + format: int64 + minimum: 0 + type: integer + oomKillGroup: + format: int64 + type: integer + priority: + format: int64 + type: integer + priorityEnable: + description: 'TODO: enhance the usages of oom priority + and oom kill group' + format: int64 + type: integer + throttlingPercent: + description: 'ThrottlingPercent specifies the throttlingFactor + percentage to calculate `memory.high` with pod memory.limits + or node allocatable memory, which triggers memcg direct + reclamation when memory usage exceeds. Lower the factor + brings more heavier reclaim pressure. Close: 0.' + format: int64 + minimum: 0 + type: integer + wmarkMinAdj: + description: 'wmark_min_adj (Anolis OS required) WmarkMinAdj + specifies `memory.wmark_min_adj` which adjusts per-memcg + threshold for global memory reclamation. Lower the factor + brings later reclamation. The adjustment uses different + formula for different value range. [-25, 0)：global_wmark_min'' + = global_wmark_min + (global_wmark_min - 0) * wmarkMinAdj + (0, 50]：global_wmark_min'' = global_wmark_min + (global_wmark_low + - global_wmark_min) * wmarkMinAdj Close: [LSR:0, LS:0, + BE:0]. Recommended: [LSR:-25, LS:-25, BE:50].' + format: int64 + maximum: 50 + minimum: -25 + type: integer + wmarkRatio: + description: 'wmark_ratio (Anolis OS required) Async memory + reclamation is triggered when cgroup memory usage exceeds + `memory.wmark_high` and the reclamation stops when usage + is below `memory.wmark_low`. Basically, `memory.wmark_high` + := min(memory.high, memory.limit_in_bytes) * memory.memory.wmark_ratio + `memory.wmark_low` := min(memory.high, memory.limit_in_bytes) + * (memory.wmark_ratio - memory.wmark_scale_factor) WmarkRatio + specifies `memory.wmark_ratio` that help calculate `memory.wmark_high`, + which triggers async memory reclamation when memory + usage exceeds. Close: 0. Recommended: 95.' + format: int64 + maximum: 100 + minimum: 0 + type: integer + wmarkScalePermill: + description: 'WmarkScalePermill specifies `memory.wmark_scale_factor` + that helps calculate `memory.wmark_low`, which stops + async memory reclamation when memory usage belows. Close: + 50. Recommended: 20.' + format: int64 + maximum: 1000 + minimum: 1 + type: integer + type: object + networkQOS: + properties: + egressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'EgressLimit describes the maximum network + bandwidth can be used in the egress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + egressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'EgressRequest describes the minimum network + bandwidth guaranteed in the egress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + enable: + type: boolean + ingressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'IngressLimit describes the maximum network + bandwidth can be used in the ingress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + ingressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'IngressRequest describes the minimum network + bandwidth guaranteed in the ingress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + type: object + resctrlQOS: + description: ResctrlQOSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object + type: object + lsrClass: + description: ResourceQOS for LSR pods. + properties: + blkioQOS: + properties: + blocks: + items: + properties: + ioCfg: + properties: + ioWeightPercent: + description: 'This field is used to set the + weight of a sub-group. Default value: 100. + Valid values: 1 to 100.' + format: int64 + maximum: 100 + minimum: 1 + type: integer + readBPS: + description: Throttling of throughput The value + is set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readIOPS: + description: Throttling of IOPS The value is + set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readLatency: + description: 'Configure the weight-based throttling + feature of blk-iocost Only used for RootClass + After blk-iocost is enabled, the kernel calculates + the proportion of requests that exceed the + read or write latency threshold out of all + requests. When the proportion is greater than + the read or write latency percentile (95%), + the kernel considers the disk to be saturated + and reduces the rate at which requests are + sent to the disk. the read latency threshold. + Unit: microseconds.' + format: int64 + type: integer + writeBPS: + format: int64 + minimum: 0 + type: integer + writeIOPS: + format: int64 + minimum: 0 + type: integer + writeLatency: + description: 'the write latency threshold. Unit: + microseconds.' + format: int64 + type: integer + type: object + name: + type: string + type: + type: string + type: object + type: array + enable: + type: boolean + type: object + cpuQOS: + description: CPUQOSCfg stores node-level config of cpu qos + properties: + coreExpeller: + description: 'whether pods of the QoS class can expel + the cgroup idle pods at the SMT-level. default = false + If set to true, pods of this QoS will use a dedicated + core sched group for noise clean with the SchedIdle + pods. NOTE: It takes effect if cpuPolicy = "coreSched".' + type: boolean + enable: + description: Enable indicates whether the cpu qos is enabled. + type: boolean + groupIdentity: + description: 'group identity value for pods, default = + 0 NOTE: It takes effect if cpuPolicy = "groupIdentity".' + format: int64 + type: integer + schedIdle: + description: 'cpu.idle value for pods, default = 0. `1` + means using SCHED_IDLE. CGroup Idle (introduced since + mainline Linux 5.15): https://lore.kernel.org/lkml/162971078674.25758.15464079371945307825.tip-bot2@tip-bot2/#r + NOTE: It takes effect if cpuPolicy = "coreSched".' + format: int64 + type: integer + type: object + memoryQOS: + description: MemoryQOSCfg stores node-level config of memory + qos + properties: + enable: + description: 'Enable indicates whether the memory qos + is enabled (default: false). This field is used for + node-level control, while pod-level configuration is + done with MemoryQOS and `Policy` instead of an `Enable` + option. Please view the differences between MemoryQOSCfg + and PodMemoryQOSConfig structs.' + type: boolean + lowLimitPercent: + description: 'LowLimitPercent specifies the lowLimitFactor + percentage to calculate `memory.low`, which TRIES BEST + protecting memory from global reclamation when memory + usage does not exceed the low limit unless no unprotected + memcg can be reclaimed. NOTE: `memory.low` should be + larger than `memory.min`. If spec.requests.memory == + spec.limits.memory, pod `memory.low` and `memory.high` + become invalid, while `memory.wmark_ratio` is still + in effect. Close: 0.' + format: int64 + minimum: 0 + type: integer + minLimitPercent: + description: 'memcg qos If enabled, memcg qos will be + set by the agent, where some fields are implicitly calculated + from pod spec. 1. `memory.min` := spec.requests.memory + * minLimitFactor / 100 (use 0 if requests.memory is + not set) 2. `memory.low` := spec.requests.memory * lowLimitFactor + / 100 (use 0 if requests.memory is not set) 3. `memory.limit_in_bytes` + := spec.limits.memory (set $node.allocatable.memory + if limits.memory is not set) 4. `memory.high` := floor[(spec.requests.memory + + throttlingFactor / 100 * (memory.limit_in_bytes or + node allocatable memory - spec.requests.memory))/pageSize] + * pageSize MinLimitPercent specifies the minLimitFactor + percentage to calculate `memory.min`, which protects + memory from global reclamation when memory usage does + not exceed the min limit. Close: 0.' + format: int64 + minimum: 0 + type: integer + oomKillGroup: + format: int64 + type: integer + priority: + format: int64 + type: integer + priorityEnable: + description: 'TODO: enhance the usages of oom priority + and oom kill group' + format: int64 + type: integer + throttlingPercent: + description: 'ThrottlingPercent specifies the throttlingFactor + percentage to calculate `memory.high` with pod memory.limits + or node allocatable memory, which triggers memcg direct + reclamation when memory usage exceeds. Lower the factor + brings more heavier reclaim pressure. Close: 0.' + format: int64 + minimum: 0 + type: integer + wmarkMinAdj: + description: 'wmark_min_adj (Anolis OS required) WmarkMinAdj + specifies `memory.wmark_min_adj` which adjusts per-memcg + threshold for global memory reclamation. Lower the factor + brings later reclamation. The adjustment uses different + formula for different value range. [-25, 0)：global_wmark_min'' + = global_wmark_min + (global_wmark_min - 0) * wmarkMinAdj + (0, 50]：global_wmark_min'' = global_wmark_min + (global_wmark_low + - global_wmark_min) * wmarkMinAdj Close: [LSR:0, LS:0, + BE:0]. Recommended: [LSR:-25, LS:-25, BE:50].' + format: int64 + maximum: 50 + minimum: -25 + type: integer + wmarkRatio: + description: 'wmark_ratio (Anolis OS required) Async memory + reclamation is triggered when cgroup memory usage exceeds + `memory.wmark_high` and the reclamation stops when usage + is below `memory.wmark_low`. Basically, `memory.wmark_high` + := min(memory.high, memory.limit_in_bytes) * memory.memory.wmark_ratio + `memory.wmark_low` := min(memory.high, memory.limit_in_bytes) + * (memory.wmark_ratio - memory.wmark_scale_factor) WmarkRatio + specifies `memory.wmark_ratio` that help calculate `memory.wmark_high`, + which triggers async memory reclamation when memory + usage exceeds. Close: 0. Recommended: 95.' + format: int64 + maximum: 100 + minimum: 0 + type: integer + wmarkScalePermill: + description: 'WmarkScalePermill specifies `memory.wmark_scale_factor` + that helps calculate `memory.wmark_low`, which stops + async memory reclamation when memory usage belows. Close: + 50. Recommended: 20.' + format: int64 + maximum: 1000 + minimum: 1 + type: integer + type: object + networkQOS: + properties: + egressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'EgressLimit describes the maximum network + bandwidth can be used in the egress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + egressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'EgressRequest describes the minimum network + bandwidth guaranteed in the egress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + enable: + type: boolean + ingressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'IngressLimit describes the maximum network + bandwidth can be used in the ingress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + ingressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'IngressRequest describes the minimum network + bandwidth guaranteed in the ingress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + type: object + resctrlQOS: + description: ResctrlQOSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object + type: object + policies: + description: Policies of pod QoS. + properties: + cpuPolicy: + description: applied policy for the CPU QoS, default = "groupIdentity" + type: string + type: object + systemClass: + description: ResourceQOS for system pods + properties: + blkioQOS: + properties: + blocks: + items: + properties: + ioCfg: + properties: + ioWeightPercent: + description: 'This field is used to set the + weight of a sub-group. Default value: 100. + Valid values: 1 to 100.' + format: int64 + maximum: 100 + minimum: 1 + type: integer + readBPS: + description: Throttling of throughput The value + is set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readIOPS: + description: Throttling of IOPS The value is + set to 0, which indicates that the feature + is disabled. + format: int64 + minimum: 0 + type: integer + readLatency: + description: 'Configure the weight-based throttling + feature of blk-iocost Only used for RootClass + After blk-iocost is enabled, the kernel calculates + the proportion of requests that exceed the + read or write latency threshold out of all + requests. When the proportion is greater than + the read or write latency percentile (95%), + the kernel considers the disk to be saturated + and reduces the rate at which requests are + sent to the disk. the read latency threshold. + Unit: microseconds.' + format: int64 + type: integer + writeBPS: + format: int64 + minimum: 0 + type: integer + writeIOPS: + format: int64 + minimum: 0 + type: integer + writeLatency: + description: 'the write latency threshold. Unit: + microseconds.' + format: int64 + type: integer + type: object + name: + type: string + type: + type: string + type: object + type: array + enable: + type: boolean + type: object + cpuQOS: + description: CPUQOSCfg stores node-level config of cpu qos + properties: + coreExpeller: + description: 'whether pods of the QoS class can expel + the cgroup idle pods at the SMT-level. default = false + If set to true, pods of this QoS will use a dedicated + core sched group for noise clean with the SchedIdle + pods. NOTE: It takes effect if cpuPolicy = "coreSched".' + type: boolean + enable: + description: Enable indicates whether the cpu qos is enabled. + type: boolean + groupIdentity: + description: 'group identity value for pods, default = + 0 NOTE: It takes effect if cpuPolicy = "groupIdentity".' + format: int64 + type: integer + schedIdle: + description: 'cpu.idle value for pods, default = 0. `1` + means using SCHED_IDLE. CGroup Idle (introduced since + mainline Linux 5.15): https://lore.kernel.org/lkml/162971078674.25758.15464079371945307825.tip-bot2@tip-bot2/#r + NOTE: It takes effect if cpuPolicy = "coreSched".' + format: int64 + type: integer + type: object + memoryQOS: + description: MemoryQOSCfg stores node-level config of memory + qos + properties: + enable: + description: 'Enable indicates whether the memory qos + is enabled (default: false). This field is used for + node-level control, while pod-level configuration is + done with MemoryQOS and `Policy` instead of an `Enable` + option. Please view the differences between MemoryQOSCfg + and PodMemoryQOSConfig structs.' + type: boolean + lowLimitPercent: + description: 'LowLimitPercent specifies the lowLimitFactor + percentage to calculate `memory.low`, which TRIES BEST + protecting memory from global reclamation when memory + usage does not exceed the low limit unless no unprotected + memcg can be reclaimed. NOTE: `memory.low` should be + larger than `memory.min`. If spec.requests.memory == + spec.limits.memory, pod `memory.low` and `memory.high` + become invalid, while `memory.wmark_ratio` is still + in effect. Close: 0.' + format: int64 + minimum: 0 + type: integer + minLimitPercent: + description: 'memcg qos If enabled, memcg qos will be + set by the agent, where some fields are implicitly calculated + from pod spec. 1. `memory.min` := spec.requests.memory + * minLimitFactor / 100 (use 0 if requests.memory is + not set) 2. `memory.low` := spec.requests.memory * lowLimitFactor + / 100 (use 0 if requests.memory is not set) 3. `memory.limit_in_bytes` + := spec.limits.memory (set $node.allocatable.memory + if limits.memory is not set) 4. `memory.high` := floor[(spec.requests.memory + + throttlingFactor / 100 * (memory.limit_in_bytes or + node allocatable memory - spec.requests.memory))/pageSize] + * pageSize MinLimitPercent specifies the minLimitFactor + percentage to calculate `memory.min`, which protects + memory from global reclamation when memory usage does + not exceed the min limit. Close: 0.' + format: int64 + minimum: 0 + type: integer + oomKillGroup: + format: int64 + type: integer + priority: + format: int64 + type: integer + priorityEnable: + description: 'TODO: enhance the usages of oom priority + and oom kill group' + format: int64 + type: integer + throttlingPercent: + description: 'ThrottlingPercent specifies the throttlingFactor + percentage to calculate `memory.high` with pod memory.limits + or node allocatable memory, which triggers memcg direct + reclamation when memory usage exceeds. Lower the factor + brings more heavier reclaim pressure. Close: 0.' + format: int64 + minimum: 0 + type: integer + wmarkMinAdj: + description: 'wmark_min_adj (Anolis OS required) WmarkMinAdj + specifies `memory.wmark_min_adj` which adjusts per-memcg + threshold for global memory reclamation. Lower the factor + brings later reclamation. The adjustment uses different + formula for different value range. [-25, 0)：global_wmark_min'' + = global_wmark_min + (global_wmark_min - 0) * wmarkMinAdj + (0, 50]：global_wmark_min'' = global_wmark_min + (global_wmark_low + - global_wmark_min) * wmarkMinAdj Close: [LSR:0, LS:0, + BE:0]. Recommended: [LSR:-25, LS:-25, BE:50].' + format: int64 + maximum: 50 + minimum: -25 + type: integer + wmarkRatio: + description: 'wmark_ratio (Anolis OS required) Async memory + reclamation is triggered when cgroup memory usage exceeds + `memory.wmark_high` and the reclamation stops when usage + is below `memory.wmark_low`. Basically, `memory.wmark_high` + := min(memory.high, memory.limit_in_bytes) * memory.memory.wmark_ratio + `memory.wmark_low` := min(memory.high, memory.limit_in_bytes) + * (memory.wmark_ratio - memory.wmark_scale_factor) WmarkRatio + specifies `memory.wmark_ratio` that help calculate `memory.wmark_high`, + which triggers async memory reclamation when memory + usage exceeds. Close: 0. Recommended: 95.' + format: int64 + maximum: 100 + minimum: 0 + type: integer + wmarkScalePermill: + description: 'WmarkScalePermill specifies `memory.wmark_scale_factor` + that helps calculate `memory.wmark_low`, which stops + async memory reclamation when memory usage belows. Close: + 50. Recommended: 20.' + format: int64 + maximum: 1000 + minimum: 1 + type: integer + type: object + networkQOS: + properties: + egressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'EgressLimit describes the maximum network + bandwidth can be used in the egress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + egressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'EgressRequest describes the minimum network + bandwidth guaranteed in the egress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + enable: + type: boolean + ingressLimit: + anyOf: + - type: integer + - type: string + default: 100 + description: 'IngressLimit describes the maximum network + bandwidth can be used in the ingress direction, unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + ingressRequest: + anyOf: + - type: integer + - type: string + default: 0 + description: 'IngressRequest describes the minimum network + bandwidth guaranteed in the ingress direction. unit: + bps(bytes per second), two expressions are supported，int + and string, int: percentage based on total bandwidth，valid + in 0-100 string: a specific network bandwidth value, + eg: 50M.' + x-kubernetes-int-or-string: true + type: object + resctrlQOS: + description: ResctrlQOSCfg stores node-level config of resctrl + qos + properties: + catRangeEndPercent: + description: LLC available range end for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + catRangeStartPercent: + description: LLC available range start for pods by percentage + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: Enable indicates whether the resctrl qos + is enabled. + type: boolean + mbaPercent: + description: MBA percent + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object + type: object + type: object + resourceUsedThresholdWithBE: + description: BE pods will be limited if node resource usage overload + properties: + cpuEvictBESatisfactionLowerPercent: + description: be.satisfactionRate = be.CPURealLimit/be.CPURequest; + be.cpuUsage = be.CPUUsed/be.CPURealLimit if be.satisfactionRate + < CPUEvictBESatisfactionLowerPercent/100 && be.usage >= CPUEvictBEUsageThresholdPercent/100, + then start to evict pod, and will evict to ${CPUEvictBESatisfactionUpperPercent} + format: int64 + type: integer + cpuEvictBESatisfactionUpperPercent: + description: be.satisfactionRate = be.CPURealLimit/be.CPURequest + if be.satisfactionRate > CPUEvictBESatisfactionUpperPercent/100, + then stop to evict. + format: int64 + type: integer + cpuEvictBEUsageThresholdPercent: + description: if be.cpuUsage >= CPUEvictBEUsageThresholdPercent/100, + then start to calculate the resources need to be released. + format: int64 + type: integer + cpuEvictPolicy: + description: 'CPUEvictPolicy defines the policy for the BECPUEvict + feature. Default: `evictByRealLimit`.' + type: string + cpuEvictTimeWindowSeconds: + description: when avg(cpuusage) > CPUEvictThresholdPercent, will + start to evict pod by cpu, and avg(cpuusage) is calculated based + on the most recent CPUEvictTimeWindowSeconds data + format: int64 + type: integer + cpuSuppressPolicy: + description: CPUSuppressPolicy + type: string + cpuSuppressThresholdPercent: + description: cpu suppress threshold percentage (0,100), default + = 65 + format: int64 + maximum: 100 + minimum: 0 + type: integer + enable: + description: whether the strategy is enabled, default = false + type: boolean + memoryEvictLowerPercent: + description: 'lower: memory release util usage under MemoryEvictLowerPercent, + default = MemoryEvictThresholdPercent - 2' + format: int64 + maximum: 100 + minimum: 0 + type: integer + memoryEvictThresholdPercent: + description: 'upper: memory evict threshold percentage (0,100), + default = 70' + format: int64 + maximum: 100 + minimum: 0 + type: integer + type: object + systemStrategy: + description: node global system config + properties: + memcgReapBackGround: + description: /sys/kernel/mm/memcg_reaper/reap_background + format: int64 + type: integer + minFreeKbytesFactor: + description: for /proc/sys/vm/min_free_kbytes, min_free_kbytes + = minFreeKbytesFactor * nodeTotalMemory /10000 + format: int64 + type: integer + totalNetworkBandwidth: + anyOf: + - type: integer + - type: string + description: 'TotalNetworkBandwidth indicates the overall network + bandwidth, cluster manager can set this field, and default value + taken from /sys/class/net/${NIC_NAME}/speed, unit: Mbps' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + watermarkScaleFactor: + description: /proc/sys/vm/watermark_scale_factor + format: int64 + type: integer + type: object + type: object + status: + description: NodeSLOStatus defines the observed state of NodeSLO + type: object + type: object + served: true + storage: true + subresources: + status: {} + +{{- end }} diff --git a/versions/v1.4.1/templates/crd/topology.node.k8s.io_noderesourcetopologies.yaml b/versions/v1.4.1/templates/crd/topology.node.k8s.io_noderesourcetopologies.yaml new file mode 100644 index 0000000..3226ffa --- /dev/null +++ b/versions/v1.4.1/templates/crd/topology.node.k8s.io_noderesourcetopologies.yaml @@ -0,0 +1,147 @@ +{{- if .Values.crds.managed }} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 + controller-gen.kubebuilder.io/version: v0.7.0 + creationTimestamp: null + name: noderesourcetopologies.topology.node.k8s.io +spec: + group: topology.node.k8s.io + names: + kind: NodeResourceTopology + listKind: NodeResourceTopologyList + plural: noderesourcetopologies + shortNames: + - node-res-topo + singular: noderesourcetopology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - topologyPolicies + - zones + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + +{{- end }} diff --git a/versions/v1.4.1/templates/koord-descheduler-config.yaml b/versions/v1.4.1/templates/koord-descheduler-config.yaml new file mode 100644 index 0000000..a05016e --- /dev/null +++ b/versions/v1.4.1/templates/koord-descheduler-config.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: koord-descheduler-config + namespace: {{ .Values.installation.namespace }} +data: + koord-descheduler-config: | + apiVersion: descheduler/v1alpha2 + kind: DeschedulerConfiguration + enableContentionProfiling: true + enableProfiling: true + healthzBindAddress: 0.0.0.0:10251 + metricsBindAddress: 0.0.0.0:10251 + leaderElection: + resourceLock: leases + resourceName: koord-descheduler + resourceNamespace: {{ .Values.installation.namespace }} + deschedulingInterval: 10s + dryRun: false + profiles: + - name: koord-descheduler + plugins: + deschedule: + disabled: + - name: "*" + # disable LowNodeLoad by default + # balance: + # enabled: + # - name: LowNodeLoad + evict: + disabled: + - name: "*" + enabled: + - name: MigrationController + pluginConfig: + - name: MigrationController + args: + apiVersion: descheduler/v1alpha2 + kind: MigrationControllerArgs + evictionPolicy: Eviction + namespaces: + exclude: + - kube-system + - {{ .Values.installation.namespace }} + evictQPS: "10" + evictBurst: 1 + - name: LowNodeLoad + args: + apiVersion: descheduler/v1alpha2 + kind: LowNodeLoadArgs + evictableNamespaces: + exclude: + - kube-system + - {{ .Values.installation.namespace }} + useDeviationThresholds: false + lowThresholds: + cpu: 45 + memory: 55 + highThresholds: + cpu: 75 + memory: 80 \ No newline at end of file diff --git a/versions/v1.4.1/templates/koord-descheduler.yaml b/versions/v1.4.1/templates/koord-descheduler.yaml new file mode 100644 index 0000000..e19f1dd --- /dev/null +++ b/versions/v1.4.1/templates/koord-descheduler.yaml @@ -0,0 +1,83 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + koord-app: koord-descheduler + name: koord-descheduler + namespace: {{ .Values.installation.namespace }} +spec: + replicas: {{ .Values.descheduler.replicas }} + selector: + matchLabels: + koord-app: koord-descheduler + minReadySeconds: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + template: + metadata: + labels: + koord-app: koord-descheduler + spec: +{{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{- toYaml . | nindent 8 }} +{{- end }} + containers: + - args: + - --v={{ .Values.descheduler.log.level }} + - --config=/config/koord-descheduler.config + command: + - /koord-descheduler + image: {{ .Values.imageRepositoryHost }}/{{ .Values.descheduler.image.repository }}:{{ .Values.descheduler.image.tag }} + imagePullPolicy: Always + name: descheduler + volumeMounts: + - mountPath: /config + name: koord-descheduler-config-volume + readinessProbe: + httpGet: + path: healthz + port: {{ .Values.descheduler.port }} + resources: + {{- toYaml .Values.descheduler.resources | nindent 12 }} + hostNetwork: {{ .Values.descheduler.hostNetwork }} + terminationGracePeriodSeconds: 10 + serviceAccountName: koord-descheduler + volumes: + - configMap: + defaultMode: 420 + items: + - key: koord-descheduler-config + path: koord-descheduler.config + name: koord-descheduler-config + name: koord-descheduler-config-volume + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: koord-app + operator: In + values: + - koord-descheduler + topologyKey: kubernetes.io/hostname + weight: 100 +{{- with .Values.descheduler.nodeAffinity }} + nodeAffinity: +{{ toYaml . | indent 10 }} +{{- end }} + +{{- if .Values.descheduler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.descheduler.nodeSelector | indent 8 }} +{{- end }} + +{{- if .Values.descheduler.tolerations }} + tolerations: +{{ toYaml .Values.descheduler.tolerations | indent 8 }} +{{- end }} + diff --git a/versions/v1.4.1/templates/koord-manager.yaml b/versions/v1.4.1/templates/koord-manager.yaml new file mode 100644 index 0000000..33b5675 --- /dev/null +++ b/versions/v1.4.1/templates/koord-manager.yaml @@ -0,0 +1,130 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + controle-plane: koordinator + name: {{ .Values.installation.namespace }} +--- +apiVersion: v1 +kind: Service +metadata: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} +spec: +{{ ( include "webhookServiceSpec" . ) | indent 2 }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: koordinator-webhook-certs + namespace: {{ .Values.installation.namespace }} +{{ ( include "webhookSecretData" . ) }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + koord-app: koord-manager + name: koord-manager + namespace: {{ .Values.installation.namespace }} +spec: + replicas: {{ .Values.manager.replicas }} + selector: + matchLabels: + koord-app: koord-manager + minReadySeconds: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + template: + metadata: + labels: + koord-app: koord-manager + spec: +{{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{- toYaml . | nindent 8 }} +{{- end }} + containers: + - args: + - --enable-leader-election + - --metrics-addr={{ .Values.manager.metrics.addr }}:{{ .Values.manager.metrics.port }} + - --health-probe-addr=:{{ .Values.manager.healthProbe.port }} + - --logtostderr=true + - --leader-election-namespace={{ .Values.installation.namespace }} + - --v={{ .Values.manager.log.level }} + - --feature-gates={{ .Values.featureGates }} + - --sync-period={{ .Values.manager.resyncPeriod }} + - --config-namespace={{ .Values.installation.namespace }} + command: + - /koord-manager + image: {{ .Values.imageRepositoryHost }}/{{ .Values.manager.image.repository }}:{{ .Values.manager.image.tag }} + imagePullPolicy: Always + name: manager + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: WEBHOOK_PORT + value: "{{ .Values.manager.webhook.port }}" + - name: WEBHOOK_CONFIGURATION_FAILURE_POLICY_PODS + value: {{ .Values.webhookConfiguration.failurePolicy.pods }} + ports: + - containerPort: {{ .Values.manager.webhook.port }} + name: webhook-server + protocol: TCP + - containerPort: {{ .Values.manager.metrics.port }} + name: metrics + protocol: TCP + - containerPort: {{ .Values.manager.healthProbe.port }} + name: health + protocol: TCP + readinessProbe: + httpGet: + path: readyz + port: {{ .Values.manager.healthProbe.port }} + resources: + {{- toYaml .Values.manager.resources | nindent 12 }} + hostNetwork: {{ .Values.manager.hostNetwork }} + terminationGracePeriodSeconds: 10 + serviceAccountName: koord-manager + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: koord-app + operator: In + values: + - koord-manager + topologyKey: kubernetes.io/hostname + weight: 100 +{{- with .Values.manager.nodeAffinity }} + nodeAffinity: +{{ toYaml . | indent 10 }} +{{- end }} + +{{- if .Values.manager.nodeSelector }} + nodeSelector: +{{ toYaml .Values.manager.nodeSelector | indent 8 }} +{{- end }} + +{{- if .Values.manager.tolerations }} + tolerations: +{{ toYaml .Values.manager.tolerations | indent 8 }} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: koord-manager +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} + namespace: {{ .Values.installation.namespace }} +{{ ( include "serviceAccountManager" . ) }} diff --git a/versions/v1.4.1/templates/koord-scheduler-config.yaml b/versions/v1.4.1/templates/koord-scheduler-config.yaml new file mode 100644 index 0000000..4920308 --- /dev/null +++ b/versions/v1.4.1/templates/koord-scheduler-config.yaml @@ -0,0 +1,132 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: koord-scheduler-config + namespace: {{ .Values.installation.namespace }} +data: + koord-scheduler-config: | + apiVersion: kubescheduler.config.k8s.io/v1beta2 + kind: KubeSchedulerConfiguration + leaderElection: + leaderElect: true + resourceLock: leases + resourceName: koord-scheduler + resourceNamespace: {{ .Values.installation.namespace }} + profiles: + - pluginConfig: + - name: NodeResourcesFit + args: + apiVersion: kubescheduler.config.k8s.io/v1beta2 + kind: NodeResourcesFitArgs + scoringStrategy: + type: LeastAllocated + resources: + - name: cpu + weight: 1 + - name: memory + weight: 1 + - name: "kubernetes.io/batch-cpu" + weight: 1 + - name: "kubernetes.io/batch-memory" + weight: 1 + - name: LoadAwareScheduling + args: + apiVersion: kubescheduler.config.k8s.io/v1beta2 + kind: LoadAwareSchedulingArgs + filterExpiredNodeMetrics: false + nodeMetricExpirationSeconds: 300 + resourceWeights: + cpu: 1 + memory: 1 + usageThresholds: + cpu: 65 + memory: 95 + # disable by default + # prodUsageThresholds indicates the resource utilization threshold of Prod Pods compared to the whole machine. + # prodUsageThresholds: + # cpu: 55 + # memory: 75 + # scoreAccordingProdUsage controls whether to score according to the utilization of Prod Pod + # scoreAccordingProdUsage: true + # aggregated supports resource utilization filtering and scoring based on percentile statistics + # aggregated: + # usageThresholds: + # cpu: 65 + # memory: 95 + # usageAggregationType: "p95" + # scoreAggregationType: "p95" + estimatedScalingFactors: + cpu: 85 + memory: 70 + - name: ElasticQuota + args: + apiVersion: kubescheduler.config.k8s.io/v1beta2 + kind: ElasticQuotaArgs + quotaGroupNamespace: {{ .Values.installation.namespace }} + plugins: + queueSort: + disabled: + - name: "*" + enabled: + - name: Coscheduling + preFilter: + enabled: + - name: Reservation + - name: NodeNUMAResource + - name: DeviceShare + - name: Coscheduling + - name: ElasticQuota + filter: + enabled: + - name: LoadAwareScheduling + - name: NodeNUMAResource + - name: DeviceShare + - name: Reservation + postFilter: + disabled: + - name: "*" + enabled: + - name: Reservation + - name: Coscheduling + - name: ElasticQuota + - name: DefaultPreemption + preScore: + enabled: + - name: Reservation # The Reservation plugin must come first + score: + enabled: + - name: LoadAwareScheduling + weight: 1 + - name: NodeNUMAResource + weight: 1 + - name: DeviceShare + weight: 1 + - name: Reservation + weight: 5000 + reserve: + enabled: + - name: Reservation # The Reservation plugin must come first + - name: LoadAwareScheduling + - name: NodeNUMAResource + - name: DeviceShare + - name: Coscheduling + - name: ElasticQuota + permit: + enabled: + - name: Coscheduling + preBind: + enabled: + - name: NodeNUMAResource + - name: DeviceShare + - name: Reservation + - name: DefaultPreBind + bind: + disabled: + - name: "*" + enabled: + - name: Reservation + - name: DefaultBinder + postBind: + enabled: + - name: Coscheduling + schedulerName: koord-scheduler \ No newline at end of file diff --git a/versions/v1.4.1/templates/koord-scheduler.yaml b/versions/v1.4.1/templates/koord-scheduler.yaml new file mode 100644 index 0000000..0bc656b --- /dev/null +++ b/versions/v1.4.1/templates/koord-scheduler.yaml @@ -0,0 +1,93 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + koord-app: koord-scheduler + name: koord-scheduler + namespace: {{ .Values.installation.namespace }} +spec: + replicas: {{ .Values.scheduler.replicas }} + selector: + matchLabels: + koord-app: koord-scheduler + minReadySeconds: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + template: + metadata: + labels: + koord-app: koord-scheduler + spec: +{{- with .Values.imagePullSecrets }} + imagePullSecrets: +{{- toYaml . | nindent 8 }} +{{- end }} + containers: + - args: + - --port={{ .Values.scheduler.port }} + - --logtostderr=true + - --authentication-skip-lookup=true + - --v={{ .Values.scheduler.log.level }} +{{- if semverCompare "= 1.22-0" .Capabilities.KubeVersion.Version }} + - --feature-gates={{ .Values.scheduler.compatible122FeatureGates }} +{{- else if semverCompare "< 1.22-0" .Capabilities.KubeVersion.Version }} + - --feature-gates={{ .Values.scheduler.compatibleBelow122FeatureGates }} +{{- else }} + - --feature-gates={{ .Values.scheduler.featureGates }} +{{- end }} + - --config=/config/koord-scheduler.config + command: + - /koord-scheduler + image: {{ .Values.imageRepositoryHost }}/{{ .Values.scheduler.image.repository }}:{{ .Values.scheduler.image.tag }} + imagePullPolicy: Always + name: scheduler + volumeMounts: + - mountPath: /config + name: koord-scheduler-config-volume + readinessProbe: + httpGet: + path: healthz + port: {{ .Values.scheduler.port }} + resources: + {{- toYaml .Values.scheduler.resources | nindent 12 }} + hostNetwork: {{ .Values.scheduler.hostNetwork }} + terminationGracePeriodSeconds: 10 + serviceAccountName: koord-scheduler + volumes: + - configMap: + defaultMode: 420 + items: + - key: koord-scheduler-config + path: koord-scheduler.config + name: koord-scheduler-config + name: koord-scheduler-config-volume + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: koord-app + operator: In + values: + - koord-scheduler + topologyKey: kubernetes.io/hostname + weight: 100 +{{- with .Values.scheduler.nodeAffinity }} + nodeAffinity: +{{ toYaml . | indent 10 }} +{{- end }} + +{{- if .Values.scheduler.nodeSelector }} + nodeSelector: +{{ toYaml .Values.scheduler.nodeSelector | indent 8 }} +{{- end }} + +{{- if .Values.scheduler.tolerations }} + tolerations: +{{ toYaml .Values.scheduler.tolerations | indent 8 }} +{{- end }} + diff --git a/versions/v1.4.1/templates/koordlet-service.yaml b/versions/v1.4.1/templates/koordlet-service.yaml new file mode 100644 index 0000000..d52e60f --- /dev/null +++ b/versions/v1.4.1/templates/koordlet-service.yaml @@ -0,0 +1,20 @@ +{{- if .Values.koordlet.enableServiceMonitor }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + koord-app: koordlet + name: koordlet-service + namespace: {{ .Values.installation.namespace }} +spec: + clusterIP: None + ports: + # TODO: Support port in values + - name: http + port: 9316 + targetPort: 9316 + selector: + koord-app: koordlet + +{{- end }} diff --git a/versions/v1.4.1/templates/koordlet-servicemonitor.yaml b/versions/v1.4.1/templates/koordlet-servicemonitor.yaml new file mode 100644 index 0000000..02e2174 --- /dev/null +++ b/versions/v1.4.1/templates/koordlet-servicemonitor.yaml @@ -0,0 +1,27 @@ +{{- if .Values.koordlet.enableServiceMonitor }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + koord-app: koordlet + name: koordlet-service-monitor + namespace: {{ .Values.installation.namespace }} +spec: + endpoints: + - interval: 30s + port: http + relabelings: + - action: replace + regex: (.*) + replacement: $1 + sourceLabels: + - __meta_kubernetes_pod_node_name + targetLabel: node + scheme: http + jobLabel: koord-app + selector: + matchLabels: + koord-app: koordlet + +{{- end }} diff --git a/versions/v1.4.1/templates/koordlet.yaml b/versions/v1.4.1/templates/koordlet.yaml new file mode 100644 index 0000000..bcaf34c --- /dev/null +++ b/versions/v1.4.1/templates/koordlet.yaml @@ -0,0 +1,160 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: koordlet + namespace: {{ .Values.installation.namespace }} + labels: + koord-app: koordlet +spec: + selector: + matchLabels: + koord-app: koordlet + minReadySeconds: 10 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 20% + template: + metadata: + labels: + koord-app: koordlet + runtimeproxy.koordinator.sh/skip-hookserver: "true" + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - command: + - /koordlet + args: + - -cgroup-root-dir=/host-cgroup/ +{{- if .Values.koordlet.features }} + - -feature-gates={{ .Values.koordlet.features }} +{{- else }} + # other feature-gates enabled by default include BECPUSuppress, CPUBurst, RdtResctrl and NodeTopologyReport + - -feature-gates=BECPUEvict=true,BEMemoryEvict=true,CgroupReconcile=true,Accelerators=true +{{- end }} + - -runtime-hooks-host-endpoint={{ .Values.koordlet.hostDirs.koordletSockDir }}/koordlet.sock + - --logtostderr=true + - --v={{ .Values.koordlet.log.level }} + image: {{ .Values.imageRepositoryHost }}/{{ .Values.koordlet.image.repository }}:{{ .Values.koordlet.image.tag }} + imagePullPolicy: Always + name: koordlet + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + resources: + {{- toYaml .Values.koordlet.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - SYS_ADMIN + privileged: true + volumeMounts: + - mountPath: /etc/localtime + name: host-time + readOnly: true + - mountPath: /host-cgroup/ + name: host-cgroup-root + - mountPath: /host-sys-fs/ + name: host-sys-fs + mountPropagation: Bidirectional + - mountPath: /host-var-run/ + name: host-var-run + readOnly: true + - mountPath: /host-run/ + name: host-run + readOnly: true + - mountPath: /host-var-run-koordlet/ + name: host-var-run-koordlet + mountPropagation: Bidirectional + - mountPath: /prediction-checkpoints + name: host-koordlet-checkpoint-dir + mountPropagation: Bidirectional + - mountPath: /host-sys/ + name: host-sys + readOnly: true + - mountPath: /etc/kubernetes/ + name: host-kubernetes + readOnly: true + - mountPath: /host-etc-hookserver/ + name: host-etc-hookserver + mountPropagation: Bidirectional + - mountPath: /var/lib/kubelet + name: host-kubelet-rootdir + readOnly: true + - mountPath: /dev + name: host-dev + mountPropagation: HostToContainer + - mountPath: /metric-data/ + name: metric-db-path + tolerations: + - operator: Exists + hostNetwork: true + hostPID: true + terminationGracePeriodSeconds: 10 + serviceAccountName: koordlet + volumes: + - hostPath: + path: /etc/localtime + type: "" + name: host-time + - hostPath: + path: /sys/fs/cgroup/ + type: "" + name: host-cgroup-root + - hostPath: + path: /sys/fs/ + type: "" + name: host-sys-fs + - hostPath: + path: /var/run/ + type: "" + name: host-var-run + - hostPath: + path: /run/ + type: "" + name: host-run + - hostPath: + path: {{ .Values.koordlet.hostDirs.koordletSockDir }} + type: "DirectoryOrCreate" + name: host-var-run-koordlet + - hostPath: + path: {{ .Values.koordlet.hostDirs.predictionCheckpointDir }} + type: "DirectoryOrCreate" + name: host-koordlet-checkpoint-dir + - hostPath: + path: /sys/ + type: "" + name: host-sys + - hostPath: + path: {{ .Values.koordlet.hostDirs.kubeletConfigDir }} + type: "" + name: host-kubernetes + - hostPath: + path: {{ .Values.koordlet.hostDirs.koordProxyRegisterDir }} + type: "" + name: host-etc-hookserver + - hostPath: + path: {{ .Values.koordlet.hostDirs.kubeletLibDir }} + type: "" + name: host-kubelet-rootdir + - hostPath: + path: /dev + type: "" + name: host-dev +{{- if .Values.koordlet.hostDirs.koordletTSDBDir }} + - hostPath: + path: {{ .Values.koordlet.hostDirs.koordletTSDBDir }} + type: "" +{{- else }} + - emptyDir: + medium: Memory + sizeLimit: 150Mi +{{- end }} + name: metric-db-path diff --git a/versions/v1.4.1/templates/priority/koord-batch.yaml b/versions/v1.4.1/templates/priority/koord-batch.yaml new file mode 100644 index 0000000..24408f4 --- /dev/null +++ b/versions/v1.4.1/templates/priority/koord-batch.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +description: Offline tasks, computing tasks, etc. +kind: PriorityClass +metadata: + name: koord-batch +preemptionPolicy: PreemptLowerPriority +value: 5000 \ No newline at end of file diff --git a/versions/v1.4.1/templates/priority/koord-free.yaml b/versions/v1.4.1/templates/priority/koord-free.yaml new file mode 100644 index 0000000..acb18fa --- /dev/null +++ b/versions/v1.4.1/templates/priority/koord-free.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +description: Run lowest-priority offline batch jobs, generally refers to not making resource budgets. +kind: PriorityClass +metadata: + name: koord-free +preemptionPolicy: PreemptLowerPriority +value: 3000 \ No newline at end of file diff --git a/versions/v1.4.1/templates/priority/koord-mid.yaml b/versions/v1.4.1/templates/priority/koord-mid.yaml new file mode 100644 index 0000000..aabf190 --- /dev/null +++ b/versions/v1.4.1/templates/priority/koord-mid.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +description: Nearline computation jobs whose SLO weaker than koord-prod. +kind: PriorityClass +metadata: + name: koord-mid +preemptionPolicy: PreemptLowerPriority +value: 7000 \ No newline at end of file diff --git a/versions/v1.4.1/templates/priority/koord-prod.yaml b/versions/v1.4.1/templates/priority/koord-prod.yaml new file mode 100644 index 0000000..b0664c3 --- /dev/null +++ b/versions/v1.4.1/templates/priority/koord-prod.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +description: Online production system for applications and businesses. +kind: PriorityClass +metadata: + name: koord-prod +preemptionPolicy: PreemptLowerPriority +value: 9000 \ No newline at end of file diff --git a/versions/v1.4.1/templates/rbac/koord-descheduler.yaml b/versions/v1.4.1/templates/rbac/koord-descheduler.yaml new file mode 100644 index 0000000..3363b3a --- /dev/null +++ b/versions/v1.4.1/templates/rbac/koord-descheduler.yaml @@ -0,0 +1,113 @@ + +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ .Values.installation.namespace }} + name: koord-descheduler +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: koord-descheduler-role +rules: +- apiGroups: + - "" + - events.k8s.io + resources: + - events + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - namespaces + verbs: + - list + - get + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - get + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - '*' +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - watch + - list +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - '*' + verbs: + - get + - list + - watch +- apiGroups: + - apps.kruise.io + resources: + - '*' + verbs: + - get + - list + - watch +- apiGroups: + - config.koordinator.sh + - slo.koordinator.sh + - scheduling.koordinator.sh + - topology.node.k8s.io + resources: + - '*' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: koord-descheduler-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: koord-descheduler-role +subjects: + - kind: ServiceAccount + name: koord-descheduler + namespace: {{ .Values.installation.namespace }} \ No newline at end of file diff --git a/versions/v1.4.1/templates/rbac/koord-manager.yaml b/versions/v1.4.1/templates/rbac/koord-manager.yaml new file mode 100644 index 0000000..e6b66d4 --- /dev/null +++ b/versions/v1.4.1/templates/rbac/koord-manager.yaml @@ -0,0 +1,254 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: koord-leader-election-role + namespace: {{ .Values.installation.namespace }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: koord-manager-role +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - config.koordinator.sh + resources: + - clustercolocationprofiles + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - get + - list + - watch + - patch + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - quota.koordinator.sh + resources: + - elasticquotaprofiles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - quota.koordinator.sh + resources: + - elasticquotaprofiles/status + verbs: + - get + - patch + - update +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch +- apiGroups: + - scheduling.koordinator.sh + resources: + - devices + verbs: + - get + - list + - watch +- apiGroups: + - scheduling.sigs.k8s.io + resources: + - elasticquotas + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - slo.koordinator.sh + resources: + - nodemetrics + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - slo.koordinator.sh + resources: + - nodemetrics/status + verbs: + - get + - patch + - update +- apiGroups: + - slo.koordinator.sh + resources: + - nodeslos + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - slo.koordinator.sh + resources: + - nodeslos/status + verbs: + - get + - patch + - update +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: koord-leader-election-rolebinding + namespace: {{ .Values.installation.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: koord-leader-election-role +subjects: + - kind: ServiceAccount + name: koord-manager + namespace: {{ .Values.installation.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: koord-manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: koord-manager-role +subjects: + - kind: ServiceAccount + name: koord-manager + namespace: {{ .Values.installation.namespace }} \ No newline at end of file diff --git a/versions/v1.4.1/templates/rbac/koord-scheduler.yaml b/versions/v1.4.1/templates/rbac/koord-scheduler.yaml new file mode 100644 index 0000000..f50daec --- /dev/null +++ b/versions/v1.4.1/templates/rbac/koord-scheduler.yaml @@ -0,0 +1,104 @@ + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: koord-scheduler-role +rules: +{{- if semverCompare "<= 1.20-0" .Capabilities.KubeVersion.Version }} +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch +{{- end }} +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - patch + - update +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - get + - list + - watch +- apiGroups: + - config.koordinator.sh + - slo.koordinator.sh + - scheduling.koordinator.sh + - topology.node.k8s.io + - scheduling.sigs.k8s.io + resources: + - '*' + verbs: + - '*' +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: koord-scheduler + namespace: {{ .Values.installation.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: koord-scheduler-rolebinding-custom +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: koord-scheduler-role +subjects: + - kind: ServiceAccount + name: koord-scheduler + namespace: {{ .Values.installation.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: koord-scheduler-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-scheduler +subjects: + - kind: ServiceAccount + name: koord-scheduler + namespace: {{ .Values.installation.namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: koord-scheduler-rolebinding-volume +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:volume-scheduler +subjects: + - kind: ServiceAccount + name: koord-scheduler + namespace: {{ .Values.installation.namespace }} \ No newline at end of file diff --git a/versions/v1.4.1/templates/rbac/koordlet.yaml b/versions/v1.4.1/templates/rbac/koordlet.yaml new file mode 100644 index 0000000..8cf1f36 --- /dev/null +++ b/versions/v1.4.1/templates/rbac/koordlet.yaml @@ -0,0 +1,69 @@ + +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ .Values.installation.namespace }} + name: koordlet +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: koordlet-role +rules: +- apiGroups: + - config.koordinator.sh + - slo.koordinator.sh + - scheduling.koordinator.sh + - topology.node.k8s.io + resources: + - '*' + verbs: + - '*' +- apiGroups: + - "" + resources: + - configmaps + - configmaps/status + verbs: + - '*' +- apiGroups: + - "" + resources: + - nodes + - nodes/status + - nodes/proxy + - pods + - pods/status + - persistentvolumeclaims + - persistentvolumeclaims/status + verbs: + - patch + - update + - get + - list + - watch +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - '*' +- apiGroups: + - "" + resources: + - events + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: koordlet-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: koordlet-role +subjects: + - kind: ServiceAccount + name: koordlet + namespace: {{ .Values.installation.namespace }} \ No newline at end of file diff --git a/versions/v1.4.1/templates/slo-controller-config.yaml b/versions/v1.4.1/templates/slo-controller-config.yaml new file mode 100644 index 0000000..f7e95fe --- /dev/null +++ b/versions/v1.4.1/templates/slo-controller-config.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: slo-controller-config + namespace: {{ .Values.installation.namespace }} +data: + colocation-config: | + { + "enable": true + } + resource-threshold-config: | + { + "clusterStrategy": { + "enable": true + } + } diff --git a/versions/v1.4.1/templates/webhookconfiguration.yaml b/versions/v1.4.1/templates/webhookconfiguration.yaml new file mode 100644 index 0000000..68026fc --- /dev/null +++ b/versions/v1.4.1/templates/webhookconfiguration.yaml @@ -0,0 +1,240 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: koordinator-mutating-webhook-configuration + annotations: + template: "" +webhooks: +{{ if contains "PodMutatingWebhook=false" .Values.featureGates }}{{ else }} +- clientConfig: + caBundle: Cg== + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /mutate-pod + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.pods }} + admissionReviewVersions: + - v1 + - v1beta1 + sideEffects: None + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} + name: mpod-create.koordinator.sh + namespaceSelector: + matchExpressions: + - key: control-plane + operator: DoesNotExist + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /mutate-pod + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.pods }} + name: mpod-update.koordinator.sh + namespaceSelector: + matchExpressions: + - key: control-plane + operator: DoesNotExist + objectSelector: + matchExpressions: + - key: pod.koordinator.sh/mutating-update + operator: In + values: + - "true" + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - UPDATE + resources: + - pods + sideEffects: None + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} +{{- end }} +{{ if contains "ElasticMutatingWebhook=false" .Values.featureGates }}{{ else }} +- clientConfig: + caBundle: Cg== + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /mutate-scheduling-sigs-k8s-io-v1alpha1-elasticquota + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.elasticquotas }} + admissionReviewVersions: + - v1 + - v1beta1 + sideEffects: None + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} + name: melasticquota.koordinator.sh + namespaceSelector: + matchExpressions: + - key: control-plane + operator: DoesNotExist + rules: + - apiGroups: + - scheduling.sigs.k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + resources: + - elasticquotas +{{- end }} +{{ if contains "NodeMutatingWebhook=true" .Values.featureGates }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /mutate-node-status + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.nodeStatus }} + name: mnode-status.koordinator.sh + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - nodes/status + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} + sideEffects: None +{{- end }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: koordinator-validating-webhook-configuration + annotations: + template: "" +webhooks: +{{- if contains "ConfigMapValidatingWebhook=true" .Values.featureGates }} +- admissionReviewVersions: + - v1beta1 + clientConfig: + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /validate-configmap + failurePolicy: Fail + name: vconfigmap.koordinator.sh + objectSelector: + matchExpressions: + - key: koordinator.sh/configmap-type + operator: In + values: + - slo-controller-config + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - configmaps + sideEffects: None +{{- end }} +{{- if contains "NodeValidatingWebhook=true" .Values.featureGates }} +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /validate-node + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.nodes }} + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} + name: vnode.koordinator.sh + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - nodes + sideEffects: None +{{- end }} +{{- if contains "PodValidatingWebhook=false" .Values.featureGates }}{{ else }} +- clientConfig: + caBundle: Cg== + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /validate-pod + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.pods }} + admissionReviewVersions: + - v1 + - v1beta1 + sideEffects: None + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} + name: vpod.koordinator.sh + namespaceSelector: + matchExpressions: + - key: control-plane + operator: DoesNotExist + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - pods +{{- end }} +{{- if contains "ElasticValidatingWebhook=false" .Values.featureGates }}{{ else }} +- clientConfig: + caBundle: Cg== + service: + name: koordinator-webhook-service + namespace: {{ .Values.installation.namespace }} + path: /validate-scheduling-sigs-k8s-io-v1alpha1-elasticquota + failurePolicy: {{ .Values.webhookConfiguration.failurePolicy.elasticquotas }} + admissionReviewVersions: + - v1 + - v1beta1 + sideEffects: None + timeoutSeconds: {{ .Values.webhookConfiguration.timeoutSeconds }} + name: velasticquota.koordinator.sh + namespaceSelector: + matchExpressions: + - key: control-plane + operator: DoesNotExist + rules: + - apiGroups: + - scheduling.sigs.k8s.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - elasticquotas +{{- end }} diff --git a/versions/v1.4.1/values.yaml b/versions/v1.4.1/values.yaml new file mode 100644 index 0000000..f0108aa --- /dev/null +++ b/versions/v1.4.1/values.yaml @@ -0,0 +1,151 @@ +# Default values for koordinator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +crds: + managed: true + +# values for koordinator installation +installation: + namespace: koordinator-system + roleListGroups: + - '*' + +featureGates: "" + +imageRepositoryHost: registry.cn-beijing.aliyuncs.com + +koordlet: + image: + repository: koordinator-sh/koordlet + tag: "v1.4.1" + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: "0" + memory: "0" + features: "" + log: + # log level for koordlet + level: "4" + hostDirs: + kubeletConfigDir: /etc/kubernetes/ + kubeletLibDir: /var/lib/kubelet/ + koordProxyRegisterDir: /etc/runtime/hookserver.d/ + koordletSockDir: /var/run/koordlet + predictionCheckpointDir: /var/run/koordlet/prediction-checkpoints + # if not specified, use tmpfs by default + koordletTSDBDir: "" + enableServiceMonitor: false + + +manager: + # settings for log print + log: + # log level for koord-manager + level: "4" + + replicas: 2 + image: + repository: koordinator-sh/koord-manager + tag: "v1.4.1" + webhook: + port: 9876 + metrics: + port: 8080 + healthProbe: + port: 8000 + + resyncPeriod: "0" + + # resources of koord-manager container + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 256Mi + + hostNetwork: false + + nodeAffinity: {} + nodeSelector: {} + tolerations: [] + +webhookConfiguration: + failurePolicy: + pods: Ignore + elasticquotas: Ignore + nodeStatus: Ignore + nodes: Ignore + timeoutSeconds: 30 + +serviceAccount: + annotations: {} + + +scheduler: + # settings for log print + log: + # log level for koord-scheduler + level: "4" + + replicas: 2 + image: + repository: koordinator-sh/koord-scheduler + tag: "v1.4.1" + port: 10251 + + # feature-gates for k8s > 1.22 + featureGates: "" + # feature-gates for k8s 1.22 + compatible122FeatureGates: "CompatibleCSIStorageCapacity=true" + # feature-gates for k8s < 1.22 + compatibleBelow122FeatureGates: "DisableCSIStorageCapacityInformer=true,CompatiblePodDisruptionBudget=true" + + # resources of koord-scheduler container + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 256Mi + + hostNetwork: false + + nodeAffinity: {} + nodeSelector: {} + tolerations: [] + +descheduler: + # settings for log print + log: + # log level for koord-descheduler + level: "4" + + replicas: 2 + image: + repository: koordinator-sh/koord-descheduler + tag: "v1.4.1" + port: 10251 + + featureGates: "" + + # resources of koord-descheduler container + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 500m + memory: 256Mi + + hostNetwork: false + + nodeAffinity: {} + nodeSelector: {} + tolerations: []