Merge pull request #4371 from wireapp/release_2024-12-11_16_35

Release 2024-12-11 - (expected chart version 5.8.0)

CHANGELOG.md

@@ -1,3 +1,69 @@
+# [2024-12-11] (Chart Release 5.8.0)
+
+## Release notes
+
+
+* [RabbitMQ events] Notifications are now also sent via RabbitMQ. Therefore RabbitMQ is now a required dependency for Cannon and Gundeck.  Cassandra is now a required dependency for Cannon and Background-Worker.  Both of them need access to the Gundeck keyspace.  These are breaking changes for Charts. (#4272, #4358, #4340)
+
+* If brig's server values config has the field `emailSMS.team`, the correct value for the personal user to team invitation URL must be set under `emailSMS.team.tExistingUserInvitationUrl`. Otherwise the URL will point to a path under the account pages and therefore a value for `externalUrls.accountPages` is required. (#4341)
+
+
+## API changes
+
+
+* The endpoint `POST /teams/:tid/invitations` gained a new optional field `allow_existing`, which controls whether an existing personal user should be invited to the team (#4336)
+
+
+## Features
+
+
+* Welcome email for new team owner. (#4333)
+
+* Added inviter's email to `GET /teams/invitation/info` endpoint. (#4332)
+
+
+## Bug fixes and other updates
+
+
+* Updated `nginz` config for personal user to team flow (#4334)
+
+* Freeze API version 7, create new dev version 8.  Also update checklist. (#4356, #4356)
+
+* Fixed config for personal user to team invitation URL template. (#4341)
+
+* Fixed search index after personal user creates team (#4362)
+
+
+## Documentation
+
+
+* Add a few more swagger descriptions and examples. (#4323)
+
+
+## Internal changes
+
+
+* `charts/wire-server-enterprise` is a Helm chart to run the `wire-server-enterprise`
+  service. This service can only be deployed with an image pull secret (the
+  registry is not open to public.) (#4359)
+
+* [Polysemy] Move email update and remove operations to effects (#4316, #4316)
+
+* Log uncaught IO exceptions in cargohold (#4352)
+
+* Updated email templates to v1.0.124 (#4328)
+
+* charts/galley: Make missing mls keys a templating error. Update MLS docs. (#4369)
+
+* [RabbitMQ events] New endpoint `GET /events` for consuming events is added (in API V8).
+
+  - When a client misses notifications because it was offline for too long, it needs to know this information so it can do a full synchronisation. This appears as the first notification in `GET /events` endpoint whenever the system detects this happening. The next acknowledgement of the message makes this notification not appear anymore until the next notification is missed. (#4272)
+  - New internal endpoint `POST /i/users/:uid/clients/:cid/consumable-notifications` is added (#4272)
+  - Connection pooling in cannon (#4348)
+  - Add consumers to the draining step on Cannon, in case of termination. (#4342)
+  - List queues more efficiently. (#4351)
+
+
 # [2024-11-04] (Chart Release 5.7.0)
 
 ## Bug fixes and other updates

Makefile

@@ -7,7 +7,7 @@ DOCKER_TAG            ?= $(USER)
 # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
 HELM_SEMVER           ?= 0.0.42
 # The list of helm charts needed on internal kubernetes testing environments
-CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana restund k8ssandra-test-cluster
+CHARTS_INTEGRATION    := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana restund k8ssandra-test-cluster wire-server-enterprise
 # The list of helm charts to publish on S3
 # FUTUREWORK: after we "inline local subcharts",
 # (e.g. move charts/brig to charts/wire-server/brig)
@@ -18,7 +18,7 @@ fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice		\
 calling-test demo-smtp elasticsearch-curator elasticsearch-external				\
 elasticsearch-ephemeral minio-external cassandra-external						\
 nginx-ingress-controller ingress-nginx-controller nginx-ingress-services reaper restund \
-k8ssandra-test-cluster ldap-scim-bridge
+k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise
 KIND_CLUSTER_NAME     := wire-server
 HELM_PARALLELISM      ?= 1 # 1 for sequential tests; 6 for all-parallel tests
 
@@ -51,7 +51,12 @@ install: init
 
 .PHONY: rabbit-clean
 rabbit-clean:
-	rabbitmqadmin -f pretty_json list queues vhost name messages | jq -r '.[] | "rabbitmqadmin delete queue name=\(.name) --vhost=\(.vhost)"' | bash
+	rabbitmqadmin -f pretty_json list queues vhost name \
+		| jq -r '.[] | "rabbitmqadmin delete queue name=\(.name) --vhost=\(.vhost)"' \
+		| bash
+	rabbitmqadmin -f pretty_json list exchanges name vhost \
+		| jq -r '.[] |select(.name | startswith("amq") | not) | select (.name != "") | "rabbitmqadmin delete exchange name=\(.name) --vhost=\(.vhost)"' \
+		| bash
 
 # Clean
 .PHONY: full-clean
@@ -134,7 +139,7 @@ crm: c db-migrate
 # Usage: TEST_INCLUDE=test1,test2 make devtest
 .PHONY: devtest
 devtest:
-	ghcid --command 'cabal repl integration' --test='Testlib.Run.mainI []'
+	ghcid --command 'cabal repl lib:integration' --test='Testlib.Run.mainI []'
 
 .PHONY: sanitize-pr
 sanitize-pr:
@@ -370,15 +375,6 @@ db-migrate: c
 libzauth:
 	$(MAKE) -C libs/libzauth install
 
-#################################
-# Useful when using Haskell IDE Engine
-# https://github.com/haskell/haskell-ide-engine
-#
-# Run this again after changes to libraries or dependencies.
-.PHONY: hie.yaml
-hie.yaml:
-	echo -e 'cradle:\n  cabal: {}' > hie.yaml
-
 #####################################
 # Today we pretend to be CI and run integration tests on kubernetes
 # (see also docs/developer/processes.md)

cassandra-schema.cql

@@ -1729,6 +1729,26 @@ CREATE TABLE gundeck_test.meta (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE gundeck_test.missed_notifications (
+    user_id uuid,
+    client_id text,
+    PRIMARY KEY (user_id, client_id)
+) WITH CLUSTERING ORDER BY (client_id ASC)
+    AND bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE gundeck_test.push (
     ptoken text,
     app text,

changelog.d/0-release-notes/4349

@@ -0,0 +1,3 @@
+* POST /scim/auth-token request body allows you to choose an IdP UUID to associate with.  If none is given, do not associate.
+
+  **WARNING:** the new behavior differs from the old one when first creating a unique SAML IdP and then the SCIM token: before this release, this request would associate the two, now it doesn't. (#4349)

changelog.d/2-features/4349

@@ -0,0 +1 @@
+* You can now create both multiple SCIM peers and multiple SAML IdPs, and freely associate them with each other (team management app implementation pending). (#4349)

charts/README.md

@@ -0,0 +1,6 @@
+# wire-server-enterprise
+
+This service contains the non-open parts of wire-server.
+
+The image registry is password protected. The credential can e.g. be provided by
+defining `secrets.configJson` with the value provided by Wire.

charts/background-worker/templates/configmap.yaml

@@ -21,6 +21,12 @@ data:
       host: federator
       port: 8080
 
+    cassandra:
+      endpoint:
+        host: {{ .cassandra.host }}
+        port: 9042
+      keyspace: gundeck
+
     {{- with .rabbitmq }}
     rabbitmq:
       host: {{ .host }}

charts/background-worker/templates/deployment.yaml

@@ -36,6 +36,11 @@ spec:
         - name: "background-worker-secrets"
           secret:
             secretName: "background-worker"
+        {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
+        - name: "background-worker-cassandra"
+          secret:
+            secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }}
+        {{- end }}
         {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
         - name: "rabbitmq-ca"
           secret:
@@ -52,6 +57,10 @@ spec:
           volumeMounts:
           - name: "background-worker-config"
             mountPath: "/etc/wire/background-worker/conf"
+          {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
+          - name: "background-worker-cassandra"
+            mountPath: "/etc/wire/background-worker/cassandra"
+          {{- end }}
           {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
           - name: "rabbitmq-ca"
             mountPath: "/etc/wire/background-worker/rabbitmq-ca/"

charts/background-worker/values.yaml

@@ -29,6 +29,8 @@ config:
     # tlsCaSecretRef:
     #   name: <secret-name>
     #   key: <ca-attribute>
+  cassandra:
+    host: aws-cassandra
 
   backendNotificationPusher:
     pushBackoffMinWait: 10000 # in microseconds, so 10ms

charts/brig/templates/configmap.yaml

@@ -186,14 +186,13 @@ data:
       {{- else }}
       {{- if .externalUrls.teamSettings }}
         tInvitationUrl: {{ .externalUrls.teamSettings }}/join/?team-code=${code}
-        tExistingUserInvitationUrl: {{ .externalUrls.teamSettings }}/accept-invitation/?team-code=${code}
       {{- else }}
         tInvitationUrl: {{ .externalUrls.nginz }}/register?team=${team}&team_code=${code}
-        tExistingUserInvitationUrl: {{ .externalUrls.nginz }}/accept-invitation/?team-code=${code}
       {{- end }}
         tActivationUrl: {{ .externalUrls.nginz }}/register?team=${team}&team_code=${code}
         tCreatorWelcomeUrl: {{ .externalUrls.teamCreatorWelcome }}
         tMemberWelcomeUrl: {{ .externalUrls.teamMemberWelcome }}
+        tExistingUserInvitationUrl: {{ .externalUrls.accountPages }}/accept-invitation/?team-code=${code}
       {{- end }}
 
     zauth:

charts/cannon/templates/configmap.yaml

@@ -1,25 +1,46 @@
 apiVersion: v1
 data:
+  {{- with .Values }}
   cannon.yaml: |
-    logFormat: {{ .Values.config.logFormat }}
-    logLevel: {{ .Values.config.logLevel }}
-    logNetStrings: {{ .Values.config.logNetStrings }}
+    logFormat: {{ .config.logFormat }}
+    logLevel: {{ .config.logLevel }}
+    logNetStrings: {{ .config.logNetStrings }}
 
     cannon:
       host: 0.0.0.0
-      port: {{ .Values.service.externalPort }}
+      port: {{ .service.externalPort }}
       externalHostFile: /etc/wire/cannon/externalHost/host.txt
 
     gundeck:
       host: gundeck
       port: 8080
 
+    cassandra:
+      endpoint:
+        host: {{ .config.cassandra.host }}
+        port: 9042
+      keyspace: gundeck
+
+    {{- with .config.rabbitmq }}
+    rabbitmq:
+      host: {{ .host }}
+      port: {{ .port }}
+      vHost: {{ .vHost }}
+      enableTls: {{ .enableTls }}
+      insecureSkipVerifyTls: {{ .insecureSkipVerifyTls }}
+      {{- if .tlsCaSecretRef }}
+      caCert: /etc/wire/cannon/rabbitmq-ca/{{ .tlsCaSecretRef.key }}
+      {{- end }}
+    {{- end }}
+
     drainOpts:
-      gracePeriodSeconds: {{ .Values.config.drainOpts.gracePeriodSeconds }}
-      millisecondsBetweenBatches: {{ .Values.config.drainOpts.millisecondsBetweenBatches }}
-      minBatchSize: {{ .Values.config.drainOpts.minBatchSize }}
+      gracePeriodSeconds: {{ .config.drainOpts.gracePeriodSeconds }}
+      millisecondsBetweenBatches: {{ .config.drainOpts.millisecondsBetweenBatches }}
+      minBatchSize: {{ .config.drainOpts.minBatchSize }}
+
+    disabledAPIVersions: {{ toJson .config.disabledAPIVersions }}
+  {{- end }}
 
-    disabledAPIVersions: {{ toJson .Values.config.disabledAPIVersions }}
 
 kind: ConfigMap
 metadata:

charts/cannon/templates/secret.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cannon
+  labels:
+    app: cannon
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  rabbitmqUsername: {{ .Values.secrets.rabbitmq.username | b64enc | quote }}
+  rabbitmqPassword: {{ .Values.secrets.rabbitmq.password | b64enc | quote }}
+

charts/cannon/templates/statefulset.yaml

@@ -92,6 +92,17 @@ spec:
 {{ toYaml .Values.resources | indent 12 }}
       {{- end }}
       - name: cannon
+        env:
+        - name: RABBITMQ_USERNAME
+          valueFrom:
+            secretKeyRef:
+              name: cannon
+              key: rabbitmqUsername
+        - name: RABBITMQ_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: cannon
+              key: rabbitmqPassword
         image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
         {{- if eq (include "includeSecurityContext" .) "true" }}
         securityContext:
@@ -102,6 +113,10 @@ spec:
           mountPath: /etc/wire/cannon/externalHost
         - name: cannon-config
           mountPath: /etc/wire/cannon/conf
+        {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
+        - name: rabbitmq-ca
+          mountPath: "/etc/wire/cannon/rabbitmq-ca/"
+        {{- end }}
         ports:
         - name: http
           containerPort: {{ .Values.service.internalPort }}
@@ -155,3 +170,8 @@ spec:
         secret:
           secretName: {{ .Values.service.nginz.tls.secretName }}
       {{- end }}
+      {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
+      - name: rabbitmq-ca
+        secret:
+          secretName: {{ .Values.config.rabbitmq.tlsCaSecretRef.name }}
+      {{- end }}

charts/cannon/values.yaml

@@ -11,6 +11,35 @@ config:
   logLevel: Info
   logFormat: StructuredJSON
   logNetStrings: false
+  rabbitmq:
+    host: rabbitmq
+    port: 5672
+    vHost: /
+    enableTls: false
+    insecureSkipVerifyTls: false
+  cassandra:
+    host: aws-cassandra
+    # To enable TLS provide a CA:
+    # tlsCa: <CA in PEM format (can be self-signed)>
+    #
+    # Or refer to an existing secret (containing the CA):
+    # tlsCaSecretRef:
+    #   name: <secret-name>
+    #   key: <ca-attribute>
+
+  redis:
+    host: redis-ephemeral-master
+    port: 6379
+    connectionMode: "master" # master | cluster
+    enableTls: false
+    insecureSkipVerifyTls: false
+    # To configure custom TLS CA, please provide one of these:
+    # tlsCa: <CA in PEM format (can be self-signed)>
+    #
+    # Or refer to an existing secret (containing the CA):
+    # tlsCaSecretRef:
+    #   name: <secret-name>
+    #   key: <ca-attribute>
 
   # See also the section 'Controlling the speed of websocket draining during
   # cannon pod replacement' in docs/how-to/install/configuration-options.rst

charts/galley/templates/secret.yaml

@@ -10,19 +10,11 @@ metadata:
 type: Opaque
 data:
   {{- if .Values.secrets.mlsPrivateKeys }}
-  {{- if .Values.secrets.mlsPrivateKeys.removal.ed25519 }}
   removal_ed25519.pem: {{ .Values.secrets.mlsPrivateKeys.removal.ed25519 | b64enc | quote }}
-  {{- end -}}
-  {{- if .Values.secrets.mlsPrivateKeys.removal.ecdsa_secp256r1_sha256 }}
   removal_ecdsa_secp256r1_sha256.pem: {{ .Values.secrets.mlsPrivateKeys.removal.ecdsa_secp256r1_sha256 | b64enc | quote }}
-  {{- end -}}
-  {{- if .Values.secrets.mlsPrivateKeys.removal.ecdsa_secp384r1_sha384 }}
   removal_ecdsa_secp384r1_sha384.pem: {{ .Values.secrets.mlsPrivateKeys.removal.ecdsa_secp384r1_sha384 | b64enc | quote }}
-  {{- end -}}
-  {{- if .Values.secrets.mlsPrivateKeys.removal.ecdsa_secp521r1_sha512 }}
   removal_ecdsa_secp521r1_sha512.pem: {{ .Values.secrets.mlsPrivateKeys.removal.ecdsa_secp521r1_sha512 | b64enc | quote }}
   {{- end -}}
-  {{- end -}}
 
   {{- if $.Values.config.enableFederation }}
   rabbitmqUsername: {{ .Values.secrets.rabbitmq.username | b64enc | quote }}