Merge branch 'main' into cassandra-make-tls-more-configurable

Signed-off-by: Tim Balzer <[email protected]>

.github/workflows/cd-pipeline.yml

@@ -67,7 +67,7 @@ jobs:
     permissions:
       contents: read
     env:
-      CSP_API_URL: https://console.cloud.vmware.com
+      CSP_API_URL: https://console.tanzu.broadcom.com
       CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }}
       VIB_PUBLIC_URL: https://cp.bromelia.vmware.com
     steps:
@@ -286,7 +286,7 @@ jobs:
     steps:
       - name: Notify in Slack channel
         if: ${{ needs.push-promotion.result != 'success' }}
-        uses: slackapi/slack-github-action@70cd7be8e40a46e8b0eced40b0de447bdb42f68e
+        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0
         with:
           channel-id: ${{ secrets.CD_SLACK_CHANNEL_ID }}
           payload: |

.github/workflows/ci-pipeline.yml

@@ -428,7 +428,7 @@ jobs:
           verification-mode: ${{ steps.get-asset-vib-config.outputs.verification_mode }}
           runtime-parameters-file: ${{ steps.get-asset-vib-config.outputs.runtime_parameters_file }}
         env:
-          CSP_API_URL: https://console.cloud.vmware.com
+          CSP_API_URL: https://console.tanzu.broadcom.com
           CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }}
           VIB_PUBLIC_URL: https://cp.bromelia.vmware.com
           # Target-Platform used by default

.vib/cilium/runtime-parameters.yaml

@@ -52,4 +52,67 @@ hubble:
       ports:
         http: 80
       type: LoadBalancer
+extraDeploy:
+- |
+  apiVersion: rbac.authorization.k8s.io/v1
+  kind: Role
+  metadata:
+    name: deployment-restart
+    namespace: "{{ include "common.names.namespace" . }}"
+  rules:
+    - apiGroups: ["apps"]
+      resources: ["deployments"]
+      verbs: ["get", "update", "patch"]
+    - apiGroups: ["apps"]
+      resources: ["daemonsets"]
+      verbs: ["get", "list", "watch"]
+    - apiGroups: [""]
+      resources: ["pods"]
+      verbs: ["get", "list"]
+  ---
+  apiVersion: rbac.authorization.k8s.io/v1
+  kind: RoleBinding
+  metadata:
+    name: deployment-restart
+    namespace: {{ include "common.names.namespace" . }}
+  subjects:
+  - kind: ServiceAccount
+    name: deployment-restarter
+    namespace: {{ include "common.names.namespace" . }}
+  roleRef:
+    kind: Role
+    name: deployment-restart
+  ---
+  apiVersion: v1
+  kind: ServiceAccount
+  metadata:
+    name: deployment-restarter
+    namespace: {{ include "common.names.namespace" . }}
+  ---
+  apiVersion: batch/v1
+  kind: Job
+  metadata:
+    name: deployment-restarter
+    namespace: {{ include "common.names.namespace" . }}
+  spec:
+    template:
+      spec:
+        automountServiceAccountToken: true
+        serviceAccountName: deployment-restarter
+        restartPolicy: OnFailure
+        containers:
+          - name: kubectl
+            image: docker.io/bitnami/kubectl:latest
+            command:
+              - /bin/bash
+            args:
+              - -ec
+              - |
+                set -o errexit
+                set -o nounset
+                set -o pipefail
 
+                kubectl rollout status --namespace {{ include "common.names.namespace" . }} daemonset {{ include "cilium.agent.fullname" . }}
+                echo "Cilium agents are ready"
+                kubectl rollout restart deployment --namespace {{ include "common.names.namespace" . }} {{ include "cilium.hubble.relay.fullname" . }}
+                kubectl rollout restart deployment --namespace {{ include "common.names.namespace" . }} {{ include "cilium.hubble.ui.fullname" . }}

.vib/clickhouse/ginkgo/clickhouse_suite_test.go

@@ -32,7 +32,7 @@ func init() {
 	flag.StringVar(&username, "username", "", "database user")
 	flag.StringVar(&password, "password", "", "database password for username")
 	flag.IntVar(&shards, "shards", 2, "number of shards")
-	flag.IntVar(&timeoutSeconds, "timeout", 180, "timeout in seconds")
+	flag.IntVar(&timeoutSeconds, "timeout", 300, "timeout in seconds")
 	timeout = time.Duration(timeoutSeconds) * time.Second
 }
 

.vib/clickhouse/ginkgo/clickhouse_test.go

@@ -10,6 +10,7 @@ import (
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
 	batchv1 "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
@@ -35,6 +36,7 @@ var _ = Describe("ClickHouse", Ordered, func() {
 
 			getAvailableReplicas := func(ss *appsv1.StatefulSet) int32 { return ss.Status.AvailableReplicas }
 			getSucceededJobs := func(j *batchv1.Job) int32 { return j.Status.Succeeded }
+			getRestartedAtAnnotation := func(pod *v1.Pod) string { return pod.Annotations["kubectl.kubernetes.io/restartedAt"] }
 			getOpts := metav1.GetOptions{}
 
 			image := ""
@@ -74,22 +76,25 @@ var _ = Describe("ClickHouse", Ordered, func() {
 				return c.BatchV1().Jobs(namespace).Get(ctx, createDBJobName, getOpts)
 			}, timeout, PollingInterval).Should(WithTransform(getSucceededJobs, Equal(int32(1))))
 
+			By("deleting the job once it has succeeded")
+			err = c.BatchV1().Jobs(namespace).Delete(ctx, createDBJobName, metav1.DeleteOptions{})
+
 			for i := 0; i < shards; i++ {
-				By(fmt.Sprintf("Scaling shard %d down to 0 replicas", i))
 				shardName := fmt.Sprintf("%s-shard%d", releaseName, i)
 				ss, err := c.AppsV1().StatefulSets(namespace).Get(ctx, shardName, getOpts)
 				shardOrigReplicas := *ss.Spec.Replicas
-				ss, err = utils.StsScale(ctx, c, ss, 0)
-				Expect(err).NotTo(HaveOccurred())
-				Expect(ss.Status.Replicas).NotTo(BeZero())
-				Eventually(func() (*appsv1.StatefulSet, error) {
-					return c.AppsV1().StatefulSets(namespace).Get(ctx, shardName, getOpts)
-				}, timeout, PollingInterval).Should(WithTransform(getAvailableReplicas, BeZero()))
 
-				By(fmt.Sprintf("Scaling shard %d to the original replicas", i))
-				ss, err = utils.StsScale(ctx, c, ss, shardOrigReplicas)
+				Expect(ss.Status.Replicas).NotTo(BeZero())
+				By("rollout restart the statefulset")
+				_, err = utils.StsRolloutRestart(ctx, c, ss)
 				Expect(err).NotTo(HaveOccurred())
 
+				for i := 0; i < int(shardOrigReplicas); i++ {
+					Eventually(func() (*v1.Pod, error) {
+						return c.CoreV1().Pods(namespace).Get(ctx, fmt.Sprintf("%s-%d", shardName, i), getOpts)
+					}, timeout, PollingInterval).Should(WithTransform(getRestartedAtAnnotation, Not(BeEmpty())))
+				}
+
 				Eventually(func() (*appsv1.StatefulSet, error) {
 					return c.AppsV1().StatefulSets(namespace).Get(ctx, shardName, getOpts)
 				}, timeout, PollingInterval).Should(WithTransform(getAvailableReplicas, Equal(shardOrigReplicas)))
@@ -104,6 +109,9 @@ var _ = Describe("ClickHouse", Ordered, func() {
 			Eventually(func() (*batchv1.Job, error) {
 				return c.BatchV1().Jobs(namespace).Get(ctx, deleteDBJobName, getOpts)
 			}, timeout, PollingInterval).Should(WithTransform(getSucceededJobs, Equal(int32(1))))
+			By("deleting the job once it has succeeded")
+			err = c.BatchV1().Jobs(namespace).Delete(ctx, deleteDBJobName, metav1.DeleteOptions{})
+			Expect(err).NotTo(HaveOccurred())
 		})
 	})
 

.vib/grafana-tempo/cypress/cypress/e2e/grafana_tempo.cy.js

@@ -26,9 +26,8 @@ it('lists all components in a healthy state', () => {
 
   cy.visit('/memberlist');
 
-  // The lower, the better. Zero --> Healthy
-  cy.contains('Health Score: 0');
-  cy.contains(`Members: ${numMembers}`);
+  cy.contains('it is healthy');
+  cy.contains(`cluster has ${numMembers} members`);
 
   // Check a random member's gossipRing port
   var randomMember = members[Math.floor(Math.random()*members.length)];

.vib/kuberay/cypress/cypress/fixtures/clusters.json

@@ -1,6 +1,6 @@
 {
   "cluster": {
     "head": "kuberay-cluster-head",
-    "worker": "kuberay-cluster-worker-vib"
+    "worker": "kuberay-cluster-vib-worker"
   }
 }

.vib/milvus/cypress/cypress/e2e/milvus.cy.js

@@ -27,7 +27,7 @@ it('allows to create a collection', () => {
     });
     // Create collection index and load
     cy.get(`[href$="${c.collection.name}${random}/overview"]`).click({force: true});
-    cy.get('span').contains('Create Index').click({force: true});
+    cy.get('button').contains('Create Index').click({force: true});
     cy.get('[data-cy="index_name"]').type(`${c.collection.idName}{enter}`);
     // Reload and wait for DOM content load
     cy.visit('#/databases');

.vib/opensearch/goss/goss.yaml

@@ -18,6 +18,23 @@ addr:
   tcp://opensearch-dashboards:{{ .Vars.dashboards.service.ports.http }}:
     reachable: true
     timeout: 180000
+http:
+  http://opensearch-master-metrics:{{ .Vars.master.metrics.service.ports.metrics }}/_prometheus/metrics:
+    status: 200
+    body:
+      - /opensearch_jvm.*opensearch-master/
+  http://opensearch-data-metrics:{{ .Vars.data.metrics.service.ports.metrics }}/_prometheus/metrics:
+    status: 200
+    body:
+      - /opensearch_jvm.*opensearch-data/
+  http://opensearch-coordinating-metrics:{{ .Vars.coordinating.metrics.service.ports.metrics }}/_prometheus/metrics:
+    status: 200
+    body:
+      - /opensearch_jvm.*opensearch-coordinating/
+  http://opensearch-ingest-metrics:{{ .Vars.ingest.metrics.service.ports.metrics }}/_prometheus/metrics:
+    status: 200
+    body:
+      - /opensearch_jvm.*opensearch-ingest/
 {{- if .Vars.master.containerSecurityContext.enabled }}
 command:
   check-no-capabilities:

.vib/opensearch/runtime-parameters.yaml

@@ -6,13 +6,33 @@ master:
   containerSecurityContext:
     enabled: true
     runAsUser: 1002
+  metrics:
+    enabled: true
+    service:
+      ports:
+        metrics: 83
 coordinating:
   replicaCount: 1
+  metrics:
+    enabled: true
+    service:
+      ports:
+        metrics: 84
 data:
   replicaCount: 2
+  metrics:
+    enabled: true
+    service:
+      ports:
+        metrics: 85
 ingest:
   enabled: true
   replicaCount: 1
+  metrics:
+    enabled: true
+    service:
+      ports:
+        metrics: 86
 service:
   type: LoadBalancer
   ports:

.vib/solr/ginkgo/solr_suite_test.go

@@ -54,7 +54,8 @@ func createJob(ctx context.Context, c kubernetes.Interface, name string, port st
 	}
 	command := []string{"solr"}
 	command = append(command, args[:]...)
-	command = append(command, "-p", port)
+	command = append(command, "--solr-url", fmt.Sprintf("http://%s:%s", stsName, port))
+
 	job := &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
@@ -72,10 +73,6 @@ func createJob(ctx context.Context, c kubernetes.Interface, name string, port st
 							Image:   image,
 							Command: command,
 							Env: []v1.EnvVar{
-								{
-									Name:  "SOLR_HOST",
-									Value: stsName,
-								},
 								{
 									Name:  "SOLR_AUTHENTICATION_OPTS",
 									Value: fmt.Sprintf("-Dbasicauth=%s:%s", username, password),

.vib/solr/goss/goss.yaml

@@ -8,14 +8,17 @@ command:
   solr-manage-collection:
     exec: |
       export {{ $auth }} && \
-      solr create_collection -p {{ .Vars.containerPorts.http }} -replicationFactor {{ $replicas }} -c {{ $collection }} && \
-      solr healthcheck -z solr-zookeeper:2181/solr -c {{ $collection }}
+      solr create_collection --solr-url http://127.0.0.1:{{ .Vars.containerPorts.http }} -replicationFactor {{ $replicas }} -c {{ $collection }}
+    # TODO: solr healthcheck is broken in version 9.7.0. In newer versions that contain the commit below it will be fixed. Uncomment and indent when it gets released
+    # https://github.com/apache/solr/commit/463e093d336ff129e7b4fbe736cc3a2bb4725d39
+    # solr healthcheck -z solr-zookeeper:2181/solr -c {{ $collection }}
     exit-status: 0
-    stdout:
-      - "healthy"
-    {{ range $e, $i := until $replicas }}
-      - /solr-{{ $i }}/
-    {{ end }}
+    # TODO: Uncomment when a new version gets released
+    # stdout:
+    #  - "healthy"
+    # {{ range $e, $i := until $replicas }}
+    #  - /solr-{{ $i }}/
+    # {{ end }}
     timeout: 30000
   {{- $uid := .Vars.containerSecurityContext.runAsUser }}
   {{- $gid := .Vars.podSecurityContext.fsGroup }}

.vib/zipkin/cypress/cypress.config.js

@@ -2,6 +2,6 @@ module.exports = {
   defaultCommandTimeout: 90000,
   e2e: {
     setupNodeEvents(on, config) {},
-    baseUrl: 'http://35.196.250.6',
+    baseUrl: 'http://localhost',
   },
 }

.vib/zipkin/cypress/cypress/e2e/zipkin.cy.js

@@ -8,11 +8,9 @@
 // We check the pushed trace in the Job
 it('should find pushed trace', () => {
   cy.visit('/')
-  cy.contains('a', 'Dependencies').click();
+  cy.get('[data-testid="add-button"]').click();
+  cy.contains('li', 'serviceName').click();
   cy.fixture('services').then((service) => {
-    cy.contains('div[class*="root"]', 'Start Time').get('input').eq(2).clear().type(service.checkTime);
-    cy.contains('button', 'Run Query').click();
-    cy.get('[role="combobox"]').click();
     cy.contains(service.name);
   });
 })

.vib/zipkin/cypress/cypress/fixtures/services.json

@@ -1,4 +1,3 @@
 {
-  "name": "yelp-main",
-  "checkTime": "09/03/2014 10:55:26"
+  "name": "yelp-main"
 }

.vib/zipkin/goss/goss.yaml

@@ -6,15 +6,15 @@ http:
     status: 200
     body:
       - /yelp-main/
-  http://zipkin:{{ .Vars.service.server.ports.http }}/api/v2/services:
+  http://zipkin:{{ .Vars.service.ports.http }}/api/v2/services:
     status: 200
     body:
       - /yelp-main/
   http://127.0.0.1:{{ .Vars.containerPorts.http }}/prometheus:
     status: 200
     body:
       - /armeria_server/
-  http://zipkin:{{ .Vars.service.server.ports.http }}/prometheus:
+  http://zipkin:{{ .Vars.service.ports.http }}/prometheus:
     status: 200
     body:
       - /armeria_server/
@@ -32,17 +32,11 @@ command:
     exit-status: 0
     stdout:
       - /AUTOCOMPLETE_TTL.*{{ .Vars.configOverrides.AUTOCOMPLETE_TTL }}/
-      {{- if not .Vars.usePasswordFile }}
-      # The secret configuration should be set via env
-      - /QUERY_NAMES_MAX_AGE.*{{ .Vars.secretConfigOverrides.QUERY_NAMES_MAX_AGE }}/
-      {{- end }}
 
-{{- if .Vars.usePasswordFile }}
 # The secret configuration should be mounted as a file
 file:
   /bitnami/zipkin/secrets/configuration/QUERY_NAMES_MAX_AGE:
     exists: true
     filetype: symlink
     contents:
       - /{{ .Vars.secretConfigOverrides.QUERY_NAMES_MAX_AGE }}/
-{{- end }}