feat: ✨ Add individual service to improve performance

Signed-off-by: Javier Salmeron Garcia <[email protected]>
bitnami · May 15, 2024 · 94b4ab5 · 94b4ab5
1 parent 0e42230
commit 94b4ab5
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 30 deletions.
diff --git a/.vib/scylladb/ginkgo/scylladb_suite_test.go b/.vib/scylladb/ginkgo/scylladb_suite_test.go
@@ -31,7 +31,7 @@ func init() {
 	flag.StringVar(&namespace, "namespace", "", "namespace where the application is running")
 	flag.StringVar(&username, "username", "", "database user")
 	flag.StringVar(&password, "password", "", "database password for username")
-	flag.IntVar(&timeoutSeconds, "timeout", 800, "timeout in seconds")
+	flag.IntVar(&timeoutSeconds, "timeout", 300, "timeout in seconds")
 	timeout = time.Duration(timeoutSeconds) * time.Second
 }
 

diff --git a/bitnami/scylladb/README.md b/bitnami/scylladb/README.md
@@ -185,6 +185,8 @@ As the image run as non-root by default, it is necessary to adjust the ownership
 | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- |
 | `replicaCount`                                      | Number of Scylladb replicas                                                                                                                                                                                       | `1`              |
 | `updateStrategy.type`                               | updateStrategy for Scylladb statefulset                                                                                                                                                                           | `RollingUpdate`  |
+| `nameResolutionThreshold`                           | Failure threshold for internal hostnames resolution                                                                                                                                                               | `5`              |
+| `nameResolutionTimeout`                             | Timeout seconds between probes for internal hostnames resolution                                                                                                                                                  | `5`              |
 | `automountServiceAccountToken`                      | Mount Service Account token in pod                                                                                                                                                                                | `false`          |
 | `hostAliases`                                       | Add deployment host aliases                                                                                                                                                                                       | `[]`             |
 | `podManagementPolicy`                               | StatefulSet pod management policy                                                                                                                                                                                 | `OrderedReady`   |
@@ -323,31 +325,34 @@ As the image run as non-root by default, it is necessary to adjust the ownership
 
 ### Traffic Exposure Parameters
 
-| Name                                    | Description                                                                        | Value       |
-| --------------------------------------- | ---------------------------------------------------------------------------------- | ----------- |
-| `service.type`                          | Scylladb service type                                                              | `ClusterIP` |
-| `service.ports.cql`                     | Scylladb service CQL Port                                                          | `9042`      |
-| `service.ports.cqlShard`                | Scylladb service CQL Port (sharded)                                                | `19042`     |
-| `service.ports.metrics`                 | Scylladb service metrics port                                                      | `8080`      |
-| `service.nodePorts.cql`                 | Node port for CQL                                                                  | `""`        |
-| `service.nodePorts.cqlShard`            | Node port for CQL (sharded)                                                        | `""`        |
-| `service.nodePorts.metrics`             | Node port for metrics                                                              | `""`        |
-| `service.extraPorts`                    | Extra ports to expose in the service (normally used with the `sidecar` value)      | `[]`        |
-| `service.loadBalancerIP`                | LoadBalancerIP if service type is `LoadBalancer`                                   | `""`        |
-| `service.loadBalancerSourceRanges`      | Service Load Balancer sources                                                      | `[]`        |
-| `service.clusterIP`                     | Service Cluster IP                                                                 | `""`        |
-| `service.externalTrafficPolicy`         | Service external traffic policy                                                    | `Cluster`   |
-| `service.annotations`                   | Provide any additional annotations which may be required.                          | `{}`        |
-| `service.sessionAffinity`               | Session Affinity for Kubernetes service, can be "None" or "ClientIP"               | `None`      |
-| `service.sessionAffinityConfig`         | Additional settings for the sessionAffinity                                        | `{}`        |
-| `service.headless.annotations`          | Annotations for the headless service.                                              | `{}`        |
-| `networkPolicy.enabled`                 | Specifies whether a NetworkPolicy should be created                                | `true`      |
-| `networkPolicy.allowExternal`           | Don't require server label for connections                                         | `true`      |
-| `networkPolicy.allowExternalEgress`     | Allow the pod to access any range of port and all destinations.                    | `true`      |
-| `networkPolicy.extraIngress`            | Add extra ingress rules to the NetworkPolicy                                       | `[]`        |
-| `networkPolicy.extraEgress`             | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true) | `[]`        |
-| `networkPolicy.ingressNSMatchLabels`    | Labels to match to allow traffic from other namespaces                             | `{}`        |
-| `networkPolicy.ingressNSPodMatchLabels` | Pod labels to match to allow traffic from other namespaces                         | `{}`        |
+| Name                                    | Description                                                                                        | Value       |
+| --------------------------------------- | -------------------------------------------------------------------------------------------------- | ----------- |
+| `service.type`                          | Scylladb service type                                                                              | `ClusterIP` |
+| `service.ports.cql`                     | Scylladb service CQL Port                                                                          | `9042`      |
+| `service.ports.cqlShard`                | Scylladb service CQL Port (sharded)                                                                | `19042`     |
+| `service.ports.metrics`                 | Scylladb service metrics port                                                                      | `8080`      |
+| `service.nodePorts.cql`                 | Node port for CQL                                                                                  | `""`        |
+| `service.nodePorts.cqlShard`            | Node port for CQL (sharded)                                                                        | `""`        |
+| `service.nodePorts.metrics`             | Node port for metrics                                                                              | `""`        |
+| `service.extraPorts`                    | Extra ports to expose in the service (normally used with the `sidecar` value)                      | `[]`        |
+| `service.loadBalancerIP`                | LoadBalancerIP if service type is `LoadBalancer`                                                   | `""`        |
+| `service.loadBalancerSourceRanges`      | Service Load Balancer sources                                                                      | `[]`        |
+| `service.clusterIP`                     | Service Cluster IP                                                                                 | `""`        |
+| `service.externalTrafficPolicy`         | Service external traffic policy                                                                    | `Cluster`   |
+| `service.annotations`                   | Provide any additional annotations which may be required.                                          | `{}`        |
+| `service.sessionAffinity`               | Session Affinity for Kubernetes service, can be "None" or "ClientIP"                               | `None`      |
+| `service.sessionAffinityConfig`         | Additional settings for the sessionAffinity                                                        | `{}`        |
+| `service.headless.annotations`          | Annotations for the headless service.                                                              | `{}`        |
+| `service.internal.enabled`              | Create a service per pod (this improves the cluster stability when scaling or performing upgrades) | `true`      |
+| `service.internal.labels`               | Labels for the internal services.                                                                  | `{}`        |
+| `service.internal.annotations`          | Annotations for the internal services.                                                             | `{}`        |
+| `networkPolicy.enabled`                 | Specifies whether a NetworkPolicy should be created                                                | `true`      |
+| `networkPolicy.allowExternal`           | Don't require server label for connections                                                         | `true`      |
+| `networkPolicy.allowExternalEgress`     | Allow the pod to access any range of port and all destinations.                                    | `true`      |
+| `networkPolicy.extraIngress`            | Add extra ingress rules to the NetworkPolicy                                                       | `[]`        |
+| `networkPolicy.extraEgress`             | Add extra ingress rules to the NetworkPolicy (ignored if allowExternalEgress=true)                 | `[]`        |
+| `networkPolicy.ingressNSMatchLabels`    | Labels to match to allow traffic from other namespaces                                             | `{}`        |
+| `networkPolicy.ingressNSPodMatchLabels` | Pod labels to match to allow traffic from other namespaces                                         | `{}`        |
 
 ### Persistence parameters
 

diff --git a/bitnami/scylladb/templates/_helpers.tpl b/bitnami/scylladb/templates/_helpers.tpl
@@ -61,8 +61,12 @@ Return the list of Scylladb seed nodes
 {{- $clusterDomain := .Values.clusterDomain }}
 {{- $seedCount := .Values.cluster.seedCount | int }}
 {{- range $e, $i := until $seedCount }}
+{{- if $.Values.service.internal.enabled -}}
+{{- $seeds = append $seeds (printf "%s-%d-internal.%s.svc.%s" $fullname $i $releaseNamespace $clusterDomain) }}
+{{- else -}}
 {{- $seeds = append $seeds (printf "%s-%d.%s-headless.%s.svc.%s" $fullname $i $fullname $releaseNamespace $clusterDomain) }}
 {{- end }}
+{{- end }}
 {{- range .Values.cluster.extraSeeds }}
 {{- $seeds = append $seeds . }}
 {{- end }}

diff --git a/bitnami/scylladb/templates/individual-svc.yaml b/bitnami/scylladb/templates/individual-svc.yaml
@@ -0,0 +1,60 @@
+{{- /*
+Copyright Broadcom, Inc. All Rights Reserved.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{/*
+These services are necessary for avoiding long startups on IP changes. This behavior
+is taken from the upstream scylla-operator
+*/}}
+
+{{- if .Values.service.internal.enabled }}
+{{- $fullName := include "common.names.fullname" . }}
+{{- $replicaCount := .Values.replicaCount | int }}
+{{- $root := . }}
+{{- range $i, $e := until $replicaCount }}
+{{- $targetPod := printf "%s-%d" (printf "%s" $fullName) $i }}
+{{- $_ := set $ "targetPod" $targetPod }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-%d-internal" $fullName $i }}
+  namespace: {{ include "common.names.namespace" $root | quote }}
+  labels: {{- include "common.labels.standard" ( dict "customLabels" $root.Values.commonLabels "context" $ ) | nindent 4 }}
+    app.kubernetes.io/part-of: scylladb
+    app.kubernetes.io/component: scylladb
+    pod: {{ $targetPod }}
+  {{- if or $root.Values.service.internal.annotations $root.Values.commonAnnotations }}
+  {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list $root.Values.service.internal.annotations $root.Values.commonAnnotations ) "context" $root ) }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  publishNotReadyAddresses: true
+  ports:
+    - name: intra
+      port: {{ $root.Values.containerPorts.intra }}
+      targetPort: intra
+    - name: tls
+      port: {{ $root.Values.containerPorts.tls }}
+      targetPort: tls
+    - name: jmx
+      port: {{ $root.Values.containerPorts.jmx }}
+      targetPort: jmx
+    - name: cql
+      port: {{ $root.Values.containerPorts.cql }}
+      targetPort: cql
+    - name: cql-shard
+      port: {{ $root.Values.containerPorts.cqlShard }}
+      targetPort: cql
+    {{- if $root.Values.service.internal.extraPorts }}
+    {{- include "common.tplvalues.render" (dict "value" $root.Values.service.internal.extraPorts "context" $) | nindent 4 }}
+    {{- end }}
+  {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list $root.Values.podLabels $root.Values.commonLabels ) "context" $root ) }}
+  selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }}
+    app.kubernetes.io/part-of: scylladb
+    app.kubernetes.io/component: scylladb
+    statefulset.kubernetes.io/pod-name: {{ $targetPod }}
+---
+  {{- end }}
+{{- end }}
diff --git a/bitnami/scylladb/templates/statefulset.yaml b/bitnami/scylladb/templates/statefulset.yaml
@@ -189,6 +189,9 @@ spec:
                   # Only node 0 will execute the startup initdb scripts
                   export SCYLLADB_IGNORE_INITDB_SCRIPTS=1
               fi
+              {{- if .Values.service.internal.enabled }}
+              export SCYLLADB_BROADCAST_ADDRESS=${POD_NAME}-internal
+              {{- end }}
               /opt/bitnami/scripts/scylladb/entrypoint.sh /opt/bitnami/scripts/scylladb/run.sh
           {{- end }}
           {{- if .Values.diagnosticMode.enabled }}
@@ -211,10 +214,6 @@ spec:
                 secretKeyRef:
                   name: {{ include "common.secrets.name" (dict "existingSecret" .Values.dbUser.existingSecret "context" $) }}
                   key: {{ include "common.secrets.key" (dict "existingSecret" .Values.dbUser.existingSecret "key" "scylladb-password") }}
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
             - name: POD_NAME
               valueFrom:
                 fieldRef:

diff --git a/bitnami/scylladb/values.yaml b/bitnami/scylladb/values.yaml
@@ -211,6 +211,12 @@ replicaCount: 1
 ##
 updateStrategy:
   type: RollingUpdate
+## @param nameResolutionThreshold Failure threshold for internal hostnames resolution
+##
+nameResolutionThreshold: 5
+## @param nameResolutionTimeout Timeout seconds between probes for internal hostnames resolution
+##
+nameResolutionTimeout: 5
 ## @param automountServiceAccountToken Mount Service Account token in pod
 ##
 automountServiceAccountToken: false
@@ -712,6 +718,18 @@ service:
     ## @param service.headless.annotations Annotations for the headless service.
     ##
     annotations: {}
+  ## Internal service properties
+  ##
+  internal:
+    ## @param service.internal.enabled Create a service per pod (this improves the cluster stability when scaling or performing upgrades)
+    ##
+    enabled: true
+    ## @param service.internal.labels Labels for the internal services.
+    ##
+    labels: {}
+    ## @param service.internal.annotations Annotations for the internal services.
+    ##
+    annotations: {}
 ## Network Policies
 ## Ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/
 ##