[CX-2073]: Starting redis instance for local dev

.gitignore

@@ -1 +1,2 @@
 .idea/
+datafold/secrets-plaintext/**

bin/dev.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+check_environment_variable() {
+  local variable_name="$1"
+
+  if [ -z "${!variable_name}" ]; then
+    echo "Error: Environment variable '$variable_name' is not set."
+    exit 1
+  fi
+}
+
+# The deploy name is a name chosen by the k8s admin to deploy this application
+check_environment_variable "DATAFOLD_DEPLOY_NAME"
+# The DB hostpath is where local postgres/redis data is stored
+# (DEV only)
+check_environment_variable "DATAFOLD_DB_HOSTPATH"
+
+# Do not use TAG yet, because that's only when we use the application container.
+# check_environment_variable "TAG"
+
+# Check if a command is provided as an argument
+if [ $# -eq 0 ]; then
+  echo "Error: Please provide a command."
+  echo "install: Installs the application."
+  echo "upgrade: Upgrades the installation."
+  echo "uninstall: Uninstalls the application."
+  exit 1
+fi
+
+# Determine which function to run based on the provided command
+case "$1" in
+  "install")
+    helm upgrade --install --namespace=datafold --create-namespace $DATAFOLD_DEPLOY_NAME ./charts/datafold --set global.hostPath=$DATAFOLD_DB_HOSTPATH
+    ;;
+  "upgrade")
+    helm upgrade --namespace=datafold --create-namespace $DATAFOLD_DEPLOY_NAME ./charts/datafold --set global.hostPath=$DATAFOLD_DB_HOSTPATH
+    ;;
+  "uninstall")
+    helm uninstall --namespace=datafold $DATAFOLD_DEPLOY_NAME
+    ;;
+  "test")
+    helm template --debug --namespace=datafold --set global.hostPath=$DATAFOLD_DB_HOSTPATH ./charts/datafold
+    ;;
+  *)
+    echo "Error: Unknown command '$1'."
+    exit 1
+    ;;
+esac

datafold/Chart.yaml → charts/datafold/Chart.yaml

@@ -12,7 +12,7 @@ maintainers:
     url: https://datafold.com
 
 dependencies:
-  - name: app
+  - name: redis
     version: "*.*.*"
-    repository: file://charts/app
-    condition: app.install
+    repository: file://charts/redis
+    condition: redis.install

charts/datafold/charts/redis/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: redis
+description: A Helm chart to run redis on Kubernetes
+type: application
+version: 0.1.0
+appVersion: "5.0-alpine"

charts/datafold/charts/redis/templates/_helpers.tpl

@@ -0,0 +1,69 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "redis.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "redis.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "redis.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "redis.labels" -}}
+helm.sh/chart: {{ include "redis.chart" . }}
+{{ include "redis.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "redis.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "redis.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "redis.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "redis.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Name of the redis volume claim
+*/}}
+{{- define "redis.pvc.name" -}}
+{{- include "redis.name" . }}-data
+{{- end -}}

charts/datafold/charts/redis/templates/persistent-volume.yaml

@@ -0,0 +1,24 @@
+{{- if not .Values.global.hostPath -}}
+# Extend with additional checks, we only want to create one volume here
+{{ fail "This version must use global.hostPath setting" }}
+{{- end -}}
+
+{{- if .Values.global.hostPath }}
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ .Values.storage.name }}
+  labels:
+    {{- include "redis.labels" . | nindent 4 }}
+spec:
+  {{- include "datafold.storageClass" . | nindent 2 }}
+  accessModes:
+    - ReadWriteOnce
+  # capacity.storage is required, but in a PV it is only a label
+  capacity:
+    storage: {{ .Values.storage.size }}
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: {{ .Values.global.hostPath }}/deploy/data/redis
+    type: Directory
+{{- end }}

charts/datafold/charts/redis/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "redis.fullname" . }}
+  labels:
+    {{- include "redis.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: 6379
+      targetPort: {{ .Values.service.port }}
+      protocol: TCP
+      name: redis
+  selector:
+    {{- include "redis.selectorLabels" . | nindent 4 }}

charts/datafold/charts/redis/templates/statefulset.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "redis.fullname" . }}
+  labels:
+    {{- include "redis.labels" . | nindent 4 }}
+spec:
+  serviceName: redis
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "redis.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "redis.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: redis
+              containerPort: 6379
+              protocol: TCP
+#          livenessProbe:
+#            exec:
+#              command:
+#              - sh
+#              - -c
+#              - redis-cli -a ping
+#            initialDelaySeconds: 10
+#            timeoutSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - name: {{ include "redis.pvc.name" . }}
+              mountPath: /data
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  volumeClaimTemplates:
+    - metadata:
+        name: {{ include "redis.pvc.name" . }}
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        {{ include "datafold.storageClass" . | nindent 8 }}
+        resources:
+          requests:
+            storage: {{ .Values.storage.size }}
+{{- if .Values.global.hostPath }}
+        volumeName: {{ .Values.storage.name }}
+{{- end }}

charts/datafold/charts/redis/templates/storage_class.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.storage.storageClassOverride }}
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ .Values.storage.name }}
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+reclaimPolicy: Retain
+{{- end }}

charts/datafold/charts/redis/values.yaml

@@ -0,0 +1,36 @@
+replicaCount: 1
+
+image:
+  repository: redis
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+storage:
+  size: 1Gi
+  name: datafold-redis-storage
+  storageClass: ""
+  storageClassOverride: false
+
+podAnnotations:
+  ad.datadoghq.com/redis.logs: '[
+        {"source": "redis",
+        "service": "redis",
+        "tags": ["env:{{ .Values.DD_ENV }}"]}
+      ]'
+
+service:
+  type: ClusterIP
+  port: 6379
+
+resources: {}
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

datafold/templates/_helpers.tpl → charts/datafold/templates/_helpers.tpl

@@ -60,3 +60,21 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+Template to derive storage class to use
+*/}}
+{{- define "datafold.storageClass" -}}
+{{- if .Values.global.hostPath -}}
+{{-     printf "storageClassName: manual" -}}
+{{- else if .Values.storage.storageClass -}}
+{{-   if (ne .Values.storage.storageClass "") -}}
+{{-     printf "storageClassName: %s" .Values.storageClass -}}
+{{-   end -}}
+{{- else if .Values.global.storageClass -}}
+{{-   if (ne .Values.global.storageClass "") -}}
+{{-     printf "storageClassName: %s" .Values.global.storageClass -}}
+{{-   end -}}
+{{- end -}}
+{{- end -}}
+

datafold/templates/configmap.yaml → charts/datafold/templates/configmap.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ datafold.fullname }}-config
+  name: {{ include "datafold.fullname" . }}-config
 data:
   PYTHONUNBUFFERED: "0"
   DATAFOLD_WEB_WORKERS: "{{ .Values.DATAFOLD_WEB_WORKERS }}"

datafold/templates/secrets.yaml → charts/datafold/templates/secrets.yaml

@@ -5,7 +5,6 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: datafold-secrets
-  namespace: medium
 type: Opaque
 data:
 {{- range $name, $secret := ($secretData) }}

datafold/values.yaml → charts/datafold/values.yaml

@@ -6,7 +6,7 @@ global:
   common:
     labels: {}
     annotations: {}
+  storageClass: ""
 
-# application specific stuff
-app:
+redis:
   install: true

helm-dep-build.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # helm-dep-build-recursive
 #
@@ -19,3 +19,5 @@ for chart in . $subcharts; do
     fi
   fi
 done
+
+echo "Finished"