From 779e20d2104d5a2186480615ce7be9869986b007 Mon Sep 17 00:00:00 2001 From: Gerard Toonstra Date: Tue, 20 Feb 2024 13:53:05 +0100 Subject: [PATCH] [CX-2425]: Use aliases for worker deployment (#52) --- charts/datafold/Chart.yaml | 42 +++++--- .../charts/worker/templates/_helpers.tpl | 2 +- .../charts/worker/templates/deployment.yaml | 5 + .../datafold/charts/worker_catalog/Chart.yaml | 6 -- .../worker_catalog/templates/_helpers.tpl | 95 ------------------- .../worker_catalog/templates/deployment.yaml | 79 --------------- .../charts/worker_catalog/templates/hpa.yaml | 32 ------- .../charts/worker_catalog/values.yaml | 52 ---------- .../charts/worker_interactive/Chart.yaml | 6 -- .../worker_interactive/templates/_helpers.tpl | 94 ------------------ .../templates/deployment.yaml | 79 --------------- .../worker_interactive/templates/hpa.yaml | 32 ------- .../charts/worker_interactive/values.yaml | 52 ---------- .../charts/worker_singletons/Chart.yaml | 6 -- .../worker_singletons/templates/_helpers.tpl | 94 ------------------ .../templates/deployment.yaml | 84 ---------------- .../worker_singletons/templates/hpa.yaml | 32 ------- .../charts/worker_singletons/values.yaml | 52 ---------- charts/datafold/values.yaml | 73 +++++++++++++- 19 files changed, 104 insertions(+), 813 deletions(-) delete mode 100644 charts/datafold/charts/worker_catalog/Chart.yaml delete mode 100644 charts/datafold/charts/worker_catalog/templates/_helpers.tpl delete mode 100644 charts/datafold/charts/worker_catalog/templates/deployment.yaml delete mode 100644 charts/datafold/charts/worker_catalog/templates/hpa.yaml delete mode 100644 charts/datafold/charts/worker_catalog/values.yaml delete mode 100644 charts/datafold/charts/worker_interactive/Chart.yaml delete mode 100644 charts/datafold/charts/worker_interactive/templates/_helpers.tpl delete mode 100644 charts/datafold/charts/worker_interactive/templates/deployment.yaml delete mode 100644 charts/datafold/charts/worker_interactive/templates/hpa.yaml delete mode 100644 charts/datafold/charts/worker_interactive/values.yaml delete mode 100644 charts/datafold/charts/worker_singletons/Chart.yaml delete mode 100644 charts/datafold/charts/worker_singletons/templates/_helpers.tpl delete mode 100644 charts/datafold/charts/worker_singletons/templates/deployment.yaml delete mode 100644 charts/datafold/charts/worker_singletons/templates/hpa.yaml delete mode 100644 charts/datafold/charts/worker_singletons/values.yaml diff --git a/charts/datafold/Chart.yaml b/charts/datafold/Chart.yaml index 5ea1226..d87d6b2 100644 --- a/charts/datafold/Chart.yaml +++ b/charts/datafold/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: datafold description: Helm chart package to deploy Datafold on kubernetes. type: application -version: 0.4.7 +version: 0.4.8 appVersion: "1.0.0" icon: https://www.datafold.com/logo.png @@ -48,20 +48,34 @@ dependencies: version: "*.*.*" repository: file://charts/worker condition: worker.install - - name: worker-catalog - version: "*.*.*" - repository: file://charts/worker_catalog - condition: worker.install - - name: worker-interactive - version: "*.*.*" - repository: file://charts/worker_interactive - condition: worker.install - - name: worker-singletons - version: "*.*.*" - repository: file://charts/worker_singletons - condition: worker.install + alias: worker + - name: worker + version: "0.1.0" + repository: file://charts/worker + condition: worker-catalog.install + alias: worker-catalog + - name: worker + version: "0.1.0" + repository: file://charts/worker + condition: worker-interactive.install + alias: worker-interactive + - name: worker + version: "0.1.0" + repository: file://charts/worker + condition: worker-singletons.install + alias: worker-singletons + - name: worker + version: "0.1.0" + repository: file://charts/worker + condition: worker-lineage.install + alias: worker-lineage + - name: worker + version: "0.1.0" + repository: file://charts/worker + condition: worker-monitors.install + alias: worker-monitor - name: nginx - version: "*.*.*" + version: "0.1.0" repository: file://charts/nginx condition: nginx.install # - name: letsencrypt diff --git a/charts/datafold/charts/worker/templates/_helpers.tpl b/charts/datafold/charts/worker/templates/_helpers.tpl index da38ef9..72e0630 100644 --- a/charts/datafold/charts/worker/templates/_helpers.tpl +++ b/charts/datafold/charts/worker/templates/_helpers.tpl @@ -35,7 +35,7 @@ Common labels */}} {{- define "worker.labels" -}} helm.sh/chart: {{ include "worker.chart" . }} -app.kubernetes.io/component: worker +app.kubernetes.io/component: {{ include "worker.name" . }} {{ include "worker.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} diff --git a/charts/datafold/charts/worker/templates/deployment.yaml b/charts/datafold/charts/worker/templates/deployment.yaml index 7fb5bd9..824d71e 100644 --- a/charts/datafold/charts/worker/templates/deployment.yaml +++ b/charts/datafold/charts/worker/templates/deployment.yaml @@ -47,6 +47,11 @@ spec: env: {{ include "worker.env" . | nindent 12 }} {{ include "datafold.env" . | nindent 12 }} + - name: DATAFOLD_FRESHPAINT_URL + valueFrom: + secretKeyRef: + name: {{ include "datafold.secrets" . }} + key: DATAFOLD_FRESHPAINT_URL livenessProbe: exec: command: diff --git a/charts/datafold/charts/worker_catalog/Chart.yaml b/charts/datafold/charts/worker_catalog/Chart.yaml deleted file mode 100644 index eb8a637..0000000 --- a/charts/datafold/charts/worker_catalog/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: worker-catalog -description: A Helm chart for Kubernetes -type: application -version: 0.1.0 -appVersion: "0.1.0" diff --git a/charts/datafold/charts/worker_catalog/templates/_helpers.tpl b/charts/datafold/charts/worker_catalog/templates/_helpers.tpl deleted file mode 100644 index e301c73..0000000 --- a/charts/datafold/charts/worker_catalog/templates/_helpers.tpl +++ /dev/null @@ -1,95 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "worker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "worker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "worker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "worker.labels" -}} -helm.sh/chart: {{ include "worker.chart" . }} -app.kubernetes.io/component: worker-catalog -{{ include "worker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -{{ include "datafold.labels" . }} -app.kubernetes.io/part-of: datafold -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "worker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "worker.name" . }} -app.kubernetes.io/part-of: datafold -{{- end }} - -{{/* -Worker queues -*/}} -{{- define "worker.env" -}} -- name: QUEUES - value: "{{ .Values.worker.queues }}" -- name: WORKERS_COUNT - value: "{{ .Values.worker.count }}" -- name: MAX_TASKS_PER_WORKER - value: "{{ .Values.worker.tasks }}" -- name: MAX_MEMORY_PER_WORKER - value: "{{ .Values.worker.memory }}" -{{- end -}} - -{{/* -Datadog annotations -*/}} -{{- define "worker.datadog.annotations" -}} -{{- if (eq .Values.global.datadog.install true) }} -ad.datadoghq.com/{{ .Chart.Name }}.logs: >- - [{ - "source": "datafold-server-onprem", - "service": "datafold-server-onprem", - "log_processing_rules": [{ - "type": "multi_line", - "name": "log_start_with_date", - "pattern" : "\\[\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\,\\d{3}\\]" - }] - }] -{{- end }} -{{- end }} - -{{/* -Datafold annotations -*/}} -{{- define "worker.datafold.annotations" -}} -replica-count: "{{ .Values.replicaCount }}" -{{- with .Values.podAnnotations }} -{{- toYaml . }} -{{- end }} -{{- end }} diff --git a/charts/datafold/charts/worker_catalog/templates/deployment.yaml b/charts/datafold/charts/worker_catalog/templates/deployment.yaml deleted file mode 100644 index 7fb5bd9..0000000 --- a/charts/datafold/charts/worker_catalog/templates/deployment.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "worker.fullname" . }} - labels: - {{- include "worker.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - {{- include "worker.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- include "worker.datafold.annotations" . | nindent 8 }} - {{- include "worker.datadog.annotations" . | nindent 8 }} - labels: - {{- include "worker.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.global.datafoldRepository }}/server:{{ .Values.global.datafoldVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: ["{{ .Values.command }}"] - envFrom: - - configMapRef: - name: {{ include "datafold.configmap" . }} - env: - {{ include "worker.env" . | nindent 12 }} - {{ include "datafold.env" . | nindent 12 }} - livenessProbe: - exec: - command: - - /bin/bash - - -c - - ps -ef | grep celeryd | grep -v grep - initialDelaySeconds: 10 - periodSeconds: 60 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/datafold/charts/worker_catalog/templates/hpa.yaml b/charts/datafold/charts/worker_catalog/templates/hpa.yaml deleted file mode 100644 index 0dbedd4..0000000 --- a/charts/datafold/charts/worker_catalog/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "worker.fullname" . }} - labels: - {{- include "worker.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "worker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/charts/datafold/charts/worker_catalog/values.yaml b/charts/datafold/charts/worker_catalog/values.yaml deleted file mode 100644 index 22bb1e2..0000000 --- a/charts/datafold/charts/worker_catalog/values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -replicaCount: 1 - -image: - pullPolicy: IfNotPresent - tag: "" - -imagePullSecrets: - - name: datafold-docker-secret -nameOverride: "" -fullnameOverride: "" -terminationGracePeriodSeconds: "30" - -command: celery_worker - -worker: - queues: "lineage,catalog" - tasks: 200 - memory: 850000 - count: 5 - -podAnnotations: {} -podLabels: {} - -podSecurityContext: - runAsUser: 1000 - runAsNonRoot: true - -securityContext: - allowPrivilegeEscalation: false - -resources: - limits: - memory: 12000Mi - requests: - cpu: 1000m - memory: 12000Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -volumes: [] - -volumeMounts: [] - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/charts/datafold/charts/worker_interactive/Chart.yaml b/charts/datafold/charts/worker_interactive/Chart.yaml deleted file mode 100644 index 27a58b7..0000000 --- a/charts/datafold/charts/worker_interactive/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: worker-interactive -description: A Helm chart for Kubernetes -type: application -version: 0.1.0 -appVersion: "0.1.0" diff --git a/charts/datafold/charts/worker_interactive/templates/_helpers.tpl b/charts/datafold/charts/worker_interactive/templates/_helpers.tpl deleted file mode 100644 index c677b47..0000000 --- a/charts/datafold/charts/worker_interactive/templates/_helpers.tpl +++ /dev/null @@ -1,94 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "worker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "worker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "worker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "worker.labels" -}} -helm.sh/chart: {{ include "worker.chart" . }} -app.kubernetes.io/component: worker-interactive -{{ include "worker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -{{ include "datafold.labels" . }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "worker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "worker.name" . }} -app.kubernetes.io/part-of: datafold -{{- end }} - -{{/* -Worker queues -*/}} -{{- define "worker.env" -}} -- name: QUEUES - value: "{{ .Values.worker.queues }}" -- name: WORKERS_COUNT - value: "{{ .Values.worker.count }}" -- name: MAX_TASKS_PER_WORKER - value: "{{ .Values.worker.tasks }}" -- name: MAX_MEMORY_PER_WORKER - value: "{{ .Values.worker.memory }}" -{{- end -}} - -{{/* -Datadog annotations -*/}} -{{- define "worker.datadog.annotations" -}} -{{- if (eq .Values.global.datadog.install true) }} -ad.datadoghq.com/{{ .Chart.Name }}.logs: >- - [{ - "source": "datafold-server-onprem", - "service": "datafold-server-onprem", - "log_processing_rules": [{ - "type": "multi_line", - "name": "log_start_with_date", - "pattern" : "\\[\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\,\\d{3}\\]" - }] - }] -{{- end }} -{{- end }} - -{{/* -Datafold annotations -*/}} -{{- define "worker.datafold.annotations" -}} -replica-count: "{{ .Values.replicaCount }}" -{{- with .Values.podAnnotations }} -{{- toYaml . }} -{{- end }} -{{- end }} diff --git a/charts/datafold/charts/worker_interactive/templates/deployment.yaml b/charts/datafold/charts/worker_interactive/templates/deployment.yaml deleted file mode 100644 index 7fb5bd9..0000000 --- a/charts/datafold/charts/worker_interactive/templates/deployment.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "worker.fullname" . }} - labels: - {{- include "worker.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - {{- include "worker.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- include "worker.datafold.annotations" . | nindent 8 }} - {{- include "worker.datadog.annotations" . | nindent 8 }} - labels: - {{- include "worker.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.global.datafoldRepository }}/server:{{ .Values.global.datafoldVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: ["{{ .Values.command }}"] - envFrom: - - configMapRef: - name: {{ include "datafold.configmap" . }} - env: - {{ include "worker.env" . | nindent 12 }} - {{ include "datafold.env" . | nindent 12 }} - livenessProbe: - exec: - command: - - /bin/bash - - -c - - ps -ef | grep celeryd | grep -v grep - initialDelaySeconds: 10 - periodSeconds: 60 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/datafold/charts/worker_interactive/templates/hpa.yaml b/charts/datafold/charts/worker_interactive/templates/hpa.yaml deleted file mode 100644 index 0dbedd4..0000000 --- a/charts/datafold/charts/worker_interactive/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "worker.fullname" . }} - labels: - {{- include "worker.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "worker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/charts/datafold/charts/worker_interactive/values.yaml b/charts/datafold/charts/worker_interactive/values.yaml deleted file mode 100644 index fea896a..0000000 --- a/charts/datafold/charts/worker_interactive/values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -replicaCount: 1 - -image: - pullPolicy: IfNotPresent - tag: "" - -imagePullSecrets: - - name: datafold-docker-secret -nameOverride: "" -fullnameOverride: "" -terminationGracePeriodSeconds: "300" - -command: celery_worker - -worker: - queues: "interactive" - tasks: 200 - memory: 850000 - count: 5 - -podAnnotations: {} -podLabels: {} - -podSecurityContext: - runAsUser: 1000 - runAsNonRoot: true - -securityContext: - allowPrivilegeEscalation: false - -resources: - limits: - memory: 6000Mi - requests: - cpu: 100m - memory: 6000Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -volumes: [] - -volumeMounts: [] - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/charts/datafold/charts/worker_singletons/Chart.yaml b/charts/datafold/charts/worker_singletons/Chart.yaml deleted file mode 100644 index ba925c1..0000000 --- a/charts/datafold/charts/worker_singletons/Chart.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v2 -name: worker-singletons -description: A Helm chart for Kubernetes -type: application -version: 0.1.0 -appVersion: "0.1.0" diff --git a/charts/datafold/charts/worker_singletons/templates/_helpers.tpl b/charts/datafold/charts/worker_singletons/templates/_helpers.tpl deleted file mode 100644 index f0d106f..0000000 --- a/charts/datafold/charts/worker_singletons/templates/_helpers.tpl +++ /dev/null @@ -1,94 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "worker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "worker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "worker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "worker.labels" -}} -helm.sh/chart: {{ include "worker.chart" . }} -app.kubernetes.io/component: worker-singletons -{{ include "worker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -{{ include "datafold.labels" . }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "worker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "worker.name" . }} -app.kubernetes.io/part-of: datafold -{{- end }} - -{{/* -Worker queues -*/}} -{{- define "worker.env" -}} -- name: QUEUES - value: "{{ .Values.worker.queues }}" -- name: WORKERS_COUNT - value: "{{ .Values.worker.count }}" -- name: MAX_TASKS_PER_WORKER - value: "{{ .Values.worker.tasks }}" -- name: MAX_MEMORY_PER_WORKER - value: "{{ .Values.worker.memory }}" -{{- end -}} - -{{/* -Datadog annotations -*/}} -{{- define "worker.datadog.annotations" -}} -{{- if (eq .Values.global.datadog.install true) }} -ad.datadoghq.com/{{ .Chart.Name }}.logs: >- - [{ - "source": "datafold-server-onprem", - "service": "datafold-server-onprem", - "log_processing_rules": [{ - "type": "multi_line", - "name": "log_start_with_date", - "pattern" : "\\[\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2}\\,\\d{3}\\]" - }] - }] -{{- end }} -{{- end }} - -{{/* -Datafold annotations -*/}} -{{- define "worker.datafold.annotations" -}} -replica-count: "{{ .Values.replicaCount }}" -{{- with .Values.podAnnotations }} -{{- toYaml . }} -{{- end }} -{{- end }} diff --git a/charts/datafold/charts/worker_singletons/templates/deployment.yaml b/charts/datafold/charts/worker_singletons/templates/deployment.yaml deleted file mode 100644 index 824d71e..0000000 --- a/charts/datafold/charts/worker_singletons/templates/deployment.yaml +++ /dev/null @@ -1,84 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "worker.fullname" . }} - labels: - {{- include "worker.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - selector: - matchLabels: - {{- include "worker.selectorLabels" . | nindent 6 }} - template: - metadata: - annotations: - {{- include "worker.datafold.annotations" . | nindent 8 }} - {{- include "worker.datadog.annotations" . | nindent 8 }} - labels: - {{- include "worker.labels" . | nindent 8 }} - {{- with .Values.podLabels }} - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.global.datafoldRepository }}/server:{{ .Values.global.datafoldVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - args: ["{{ .Values.command }}"] - envFrom: - - configMapRef: - name: {{ include "datafold.configmap" . }} - env: - {{ include "worker.env" . | nindent 12 }} - {{ include "datafold.env" . | nindent 12 }} - - name: DATAFOLD_FRESHPAINT_URL - valueFrom: - secretKeyRef: - name: {{ include "datafold.secrets" . }} - key: DATAFOLD_FRESHPAINT_URL - livenessProbe: - exec: - command: - - /bin/bash - - -c - - ps -ef | grep celeryd | grep -v grep - initialDelaySeconds: 10 - periodSeconds: 60 - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/datafold/charts/worker_singletons/templates/hpa.yaml b/charts/datafold/charts/worker_singletons/templates/hpa.yaml deleted file mode 100644 index 0dbedd4..0000000 --- a/charts/datafold/charts/worker_singletons/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "worker.fullname" . }} - labels: - {{- include "worker.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "worker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/charts/datafold/charts/worker_singletons/values.yaml b/charts/datafold/charts/worker_singletons/values.yaml deleted file mode 100644 index 7a1b2ed..0000000 --- a/charts/datafold/charts/worker_singletons/values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -replicaCount: 1 - -image: - pullPolicy: IfNotPresent - tag: "" - -imagePullSecrets: - - name: datafold-docker-secret -nameOverride: "" -fullnameOverride: "" -terminationGracePeriodSeconds: "30" - -command: celery_worker - -worker: - queues: "singletons" - tasks: 200 - memory: 850000 - count: 4 - -podAnnotations: {} -podLabels: {} - -podSecurityContext: - runAsUser: 1000 - runAsNonRoot: true - -securityContext: - allowPrivilegeEscalation: false - -resources: - limits: - memory: 5000Mi - requests: - cpu: 100m - memory: 5000Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -volumes: [] - -volumeMounts: [] - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/charts/datafold/values.yaml b/charts/datafold/values.yaml index 2a94bfe..ffeef7b 100644 --- a/charts/datafold/values.yaml +++ b/charts/datafold/values.yaml @@ -73,9 +73,6 @@ server: scheduler: install: true -worker: - install: true - nginx: install: true @@ -138,3 +135,73 @@ secrets: frontendToken: "" url: "https://api.perfalytics.com/track" avoToken: "" + +worker: + install: true + replicaCount: 1 + terminationGracePeriodSeconds: "1800" + worker: + queues: "celery,api,ci,interactive,freshpaint,alerts" + tasks: 200 + memory: 850000 + count: 15 + resources: + limits: + memory: 9000Mi + requests: + cpu: 1000m + memory: 9000Mi + +worker-catalog: + install: true + replicaCount: 1 + terminationGracePeriodSeconds: "30" + worker: + queues: "lineage,catalog" + tasks: 200 + memory: 850000 + count: 5 + resources: + limits: + memory: 12000Mi + requests: + cpu: 1000m + memory: 12000Mi + +worker-interactive: + install: true + replicaCount: 1 + terminationGracePeriodSeconds: "300" + worker: + queues: "interactive" + tasks: 200 + memory: 850000 + count: 5 + resources: + limits: + memory: 6000Mi + requests: + cpu: 100m + memory: 6000Mi + +worker-singletons: + install: true + replicaCount: 1 + terminationGracePeriodSeconds: "30" + worker: + queues: "singletons" + tasks: 200 + memory: 850000 + count: 4 + resources: + limits: + memory: 5000Mi + requests: + cpu: 100m + memory: 5000Mi + +worker-lineage: + install: false + +worker-monitors: + install: false