Merge pull request #1385 from wireapp/release_2021_02_25

## Bug fixes and other updates

* Pin kubectl image in sftd chart (#1383)
* Remove imagePullPolicy: Always for reaper chart (#1387)


## Internal changes

* Use mu-haskell to implement one initial federation request across backends (#1319)
* Add migrate-external-ids tool (#1384)

.gitignore

@@ -92,3 +92,8 @@ hie.yaml
 .stack-root-buildah/
 .stack-work-buildah/
 dist-buildah
+
+# federation tests
+i.yaml
+b.yaml
+telepresence.log

CHANGELOG.md

@@ -13,6 +13,18 @@
 ## Internal changes
 
 -->
+# [2020-02-25]
+
+## Bug fixes and other updates
+
+* Pin kubectl image in sftd chart (#1383)
+* Remove imagePullPolicy: Always for reaper chart (#1387)
+
+
+## Internal changes
+
+* Use mu-haskell to implement one initial federation request across backends (#1319)
+* Add migrate-external-ids tool (#1384)
 
 # [2020-02-16]
 
@@ -50,7 +62,7 @@ This release might require manual migration steps, see [ElasticSearch migration
 * Add missing internal qa routes (#1336)
 * Extract and rename PolyLog to a library for reusability (#1329)
 * Fix: Spar integration tests misconfigured on CI (#1343)
-* Bump ormolu version (#1366, #1368) 
+* Bump ormolu version (#1366, #1368)
 * Update ES upgrade path (#1339) (#1376)
 * Bump saml2-web-sso version to latest upstream (#1369)
 * Add docs for deriving-swagger2 (#1373)

Makefile

@@ -244,29 +244,34 @@ hie.yaml:
 #   - kubectl
 #   - a valid kubectl context configured (i.e. access to a kubernetes cluster)
 .PHONY: kube-integration
-kube-integration: guard-tag charts-integration
-	# by default "test-<your computer username> is used as namespace
-	# you can override the default by setting the NAMESPACE environment variable
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup.sh
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh
+kube-integration:  kube-integration-setup kube-integration-test
 
 .PHONY: kube-integration-setup
-kube-integration-setup: guard-tag charts-integration
+kube-integration-setup: charts-integration
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh
+
+.PHONY: kube-integration-test
+kube-integration-test:
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-test.sh
+
+.PHONY: kube-integration-teardown
+kube-integration-teardown:
+	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown-federation.sh
+
+.PHONY: kube-integration-setup-sans-federation
+kube-integration-setup-sans-federation: guard-tag charts-integration
 	# by default "test-<your computer username> is used as namespace
 	# you can override the default by setting the NAMESPACE environment variable
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup.sh
 
-.PHONY: kube-integration-teardown
-kube-integration-teardown:
+.PHONY: kube-integration-teardown-sans-federation
+kube-integration-teardown-sans-federation:
 	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-teardown.sh
 
-.PHONY: kube-integration-setup-federation
-kube-integration-setup-federation: guard-tag charts-integration
-	export NAMESPACE=$(NAMESPACE); ./hack/bin/integration-setup-federation.sh
-
-.PHONY: kube-integration-federation
-kube-integration-federation:
-	cd services/brig && ./federation-tests.sh $(NAMESPACE)
+.PHONY: kube-restart-%
+kube-restart-%:
+	kubectl delete pod -n $(NAMESPACE) -l wireService=$(*)
+	kubectl delete pod -n $(NAMESPACE)-fed2 -l wireService=$(*)
 
 .PHONY: latest-brig-tag
 latest-brig-tag:

charts/brig/templates/tests/brig-integration.yaml

@@ -53,7 +53,7 @@ spec:
     # same file-system.
     # The other test, "user.auth.cookies.limit", is skipped as it is flaky.
     # This is tracked in https://github.com/zinfra/backend-issues/issues/1150.
-    command: ["brig-integration", "--pattern", "!/turn/ && !/user.auth.cookies.limit/ && !/brig-federation/"]
+    command: [ "brig-integration", "--pattern", "!/turn/ && !/user.auth.cookies.limit/" ]
     volumeMounts:
     - name: "brig-integration"
       mountPath: "/etc/wire/integration"
@@ -79,4 +79,8 @@ spec:
       value: "dummy"
     - name: AWS_REGION
       value: "eu-west-1"
+    {{- if .Values.tests.enableFederationTests }}
+    - name: INTEGRATION_FEDERATION_TESTS
+      value: "1"
+    {{- end }}
   restartPolicy: Never

charts/brig/values.yaml

@@ -87,3 +87,5 @@ turnStatic:
   v2:
   - turn:localhost:3478
   - turn:localhost:3478?transport=tcp
+tests:
+  enableFederationTests: false

charts/federator/templates/configmap.yaml

@@ -34,21 +34,20 @@ data:
     logFormat: {{ .logFormat }}
     logLevel: {{ .logLevel }}
 
-
     {{- with .optSettings }}
     optSettings:
       setFederationStrategy:
         {{- if .setFederationStrategy.allowAll }}
         allowAll:
         {{- else if .setFederationStrategy.allowedDomains }}
         allowedDomains:
-          {{- with $domain := .setFederationStrategy.allowedDomains }}
+        {{- range $domain := .setFederationStrategy.allowedDomains }}
           - {{ $domain | quote }}
-          {{- end }}
+        {{- end }}
         {{- else }}
         # In gotemplate there is no way to distinguish between empty list and no
         # list, we assume empty list when there is no list
         allowedDomains: []
         {{- end}}
     {{- end }}
-  {{- end }}
+    {{- end }}

charts/federator/templates/deployment.yaml

@@ -34,7 +34,7 @@ spec:
       containers:
         - name: federator
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: {{ default "" .Values.image.pullPolicy | quote }}
+          imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
           volumeMounts:
             - name: "federator-config"
               mountPath: "/etc/wire/federator/conf"

charts/federator/templates/service.yaml

@@ -14,7 +14,7 @@ spec:
       port: {{ .Values.service.internalFederatorPort }}
       targetPort: {{ .Values.service.internalFederatorPort }}
 
-    - name: external
+    - name: wire-server-federator
       port: {{ .Values.service.externalFederatorPort }}
       targetPort: {{ .Values.service.externalFederatorPort }}
   selector:

charts/federator/values.yaml

@@ -17,7 +17,7 @@ resources:
     memory: "512Mi"
     cpu: "500m"
 config:
-  logLevel: Info
+  logLevel: Debug
   logFormat: JSON
   optSettings:
     setFederationStrategy:

charts/reaper/templates/deployment.yaml

@@ -22,7 +22,6 @@ spec:
       serviceAccountName: reaper-role
       containers:
       - name: reaper
-        imagePullPolicy: Always
         image: bitnami/kubectl:1.19.7
         command: ["bash"]
         args:

charts/sftd/templates/statefulset.yaml

@@ -36,7 +36,7 @@ spec:
           emptyDir: {}
       initContainers:
         - name: get-external-ip
-          image: bitnami/kubectl
+          image: bitnami/kubectl:1.19.7
           volumeMounts:
             - name: external-ip
               mountPath: /external-ip

docs/reference/config-options.md

@@ -84,3 +84,74 @@ values](https://github.com/wireapp/wire-server/blob/custom-search-visibility-lim
 ### Email Visibility
 
 [Allowd values](https://github.com/wireapp/wire-server/blob/0126651a25aabc0c5589edc2b1988bb06550a03a/services/brig/src/Brig/Options.hs#L304-L306) and their [description](https://github.com/wireapp/wire-server/blob/0126651a25aabc0c5589edc2b1988bb06550a03a/services/brig/src/Brig/Options.hs#L290-L299).
+
+
+### Federation Domain
+
+Regardless of whether a backend wants to enable federation or not, the operator
+must decide what its domain is going to be. This helps in keeping things
+simpler across all components of Wire and also enables to turn on federation in
+the future if required.
+
+For production uses, it is highly recommended that this domain be configured as
+something that is controlled by the operator(s). The backend or frontend do not
+need to be available on this domain. As per our current federation design, you
+must be able to set an SRV record for `_wire-server-federator._tcp.<domain>`.
+This record should have entries which lead to the federator.
+
+**IMPORTANT** Once this option is set, it cannot be changed without breaking
+experience for all the users which are already using the backend.
+
+This configuration needs to be made in brig and in galley. (note the slighly different spelling of the config options)
+
+```yaml
+# galley.yaml
+settings:
+  federationDomain: example.com
+```
+
+```yaml
+# brig.yaml
+optSettings:
+  setFederationDomain: example.com
+```
+
+### Federation allow list
+
+As of 2021-02, federation (whatever is implemented by the time you read this) is turned off by default by means of having an empty allow list:
+
+```yaml
+# federator.yaml
+optSettings:
+  setFederationStrategy:
+    allowedDomains: []
+```
+
+You can choose to federate with a specific list of allowed servers:
+
+
+```yaml
+# federator.yaml
+optSettings:
+  setFederationStrategy:
+    allowedDomains:
+      - server1.example.com
+      - server2.example.com
+```
+
+or, you can federate with everyone:
+
+```yaml
+# federator.yaml
+optSettings:
+  setFederationStrategy:
+    # note the 'empty' value after 'allowAll'
+    allowAll:
+
+# when configuring helm charts, this becomes (note 'true' after 'allowAll')
+# inside helm_vars/wire-server:
+federator:
+  optSettings:
+    setFederationStrategy:
+      allowAll: true
+```

docs/reference/federation/README.md

@@ -0,0 +1,3 @@
+# Federation
+
+This folder contains some developer reference documentation on federation. It's early days, and more documentation may be added here, or elsewhere as time goes on.

docs/reference/federation/img/remote_user_handle_lookup.swimlanes.io.txt

@@ -0,0 +1,21 @@
+// copy the following to use with https://swimlanes.io
+
+title: PR #1319: remote user handle lookup
+
+Wire client -> brig@A: handle=alice domain=example.com
+
+brig@A -> federator@A: Outward.call(FederatedRequest(example.com, Request(..)))
+
+note federator@A,federator@B: In the future, requests between backends here should be using some server2server authentication and also make use of an authorization strategy (open federation, allow list, ...).
+
+federator@A -> federator@B: Inward.call(Request(brig, "/handle/alice"))
+
+federator@B -> brig@B: Request(/handle/alice)
+
+brig@B -> federator@B: Response
+
+federator@B -> federator@A: Response
+
+federator@A -> brig@A: Response
+
+brig@A -> Wire client: userId=1234 domain=example.com

docs/reference/federation/pull-requests/1319_initial_federation_request_across_backends.md

@@ -0,0 +1,30 @@
+This PR #1319 is one small piece in the context of Federation, namely code for the first few bytes to travel from one backend to another backend. The example implemented here is exact handle search as per https://wearezeta.atlassian.net/browse/SQCORE-108
+
+For general context about federation design, see (sorry non-Wire employees, this documentation will eventually be made available) https://github.com/wearezeta/documentation/blob/master/topics/federation/federation-design.md
+
+This introduces the [mu-haskell](https://higherkindness.io/mu-haskell/) set of libraries as a new stack dependency to gain support in wire-server for http2 / GRPC (based on protobuf).
+
+We make use of GRPC as a protocol as a new intra-service call internally from `brig` to `federator` (for the reverse flow from `federator` to `brig` we still use a REST API for the time being to not change too much at once), and also as a protocol between backends.
+
+The following networking flow is implemented:
+
+(to modify this diagram, see the swimlanes.io [source](../img/remote_user_handle_lookup.swimlanes.io.txt))
+
+![remote_user_handle_lookup](../img/remote_user_handle_lookup.png)
+
+1. A component (e.g. 'brig') will send some data of type 'FederatedRequest' to
+the 'federator' server (more precisely: the 'Outward' service part of the federator) within a same private network.
+2. The federator will use the domain from the 'FederatedRequest' to discover
+where to send some data of type 'Request' to.
+3. On the other side, a publicly exposed 'Inward' service (also co-hosted on the
+federator) will turn the received 'Request' into a 'Response' by making a call to a
+component (e.g. brig) on its private network.
+
+See also [wire-api-federation/proto/router.proto](../../../../libs/wire-api-federation/proto/router.proto) for details of the grpc protocol.
+
+Note: Server-server authentication is not yet implemented, so this code uses plain TCP between different backends at this point. This should be okay for the time being, since:
+- the federator component is disabled by default in the wire-server helm chart and won't be "accidentally" installed
+- we do not, and do not plan to run the federator in staging or production for the time being until much more of federation has been implemented: any request from brig to federator thus always fails with a 404 or other error client-side at this point.
+- (instead we will have separate federation playground servers)
+
+The core types allowing an easy extension of this code to support other requests and endpoints than the handle lookup can be found in libs/wire-server-api (notably the `proto/router.proto` file).

hack/bin/integration-setup-federation.sh

@@ -11,7 +11,9 @@ export NAMESPACE=${NAMESPACE:-test-integration}
 
 $DIR/integration-setup.sh
 
-# The suffix '-fed2' must be kept in sync with configuration inside charts/brig/templates/tests/configmap.yaml
+# The suffix '-fed2' must be kept in sync with configuration inside
+# charts/brig/templates/tests/configmap.yaml and
+# hack/bin/integration-teardown-federation.sh
 export NAMESPACE=${NAMESPACE}-fed2
 
 $DIR/integration-setup.sh

hack/bin/integration-setup.sh

@@ -36,7 +36,7 @@ function printLogs() {
 
 trap printLogs ERR
 
-FEDERATION_DOMAIN="$NAMESPACE.svc.cluster.local"
+FEDERATION_DOMAIN="federator.$NAMESPACE.svc.cluster.local"
 
 for chart in "${charts[@]}"; do
     kubectl -n ${NAMESPACE} get pods

hack/bin/integration-teardown-federation.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export NAMESPACE=${NAMESPACE:-test-integration}
+
+$DIR/integration-teardown.sh
+
+# The suffix '-fed2' must be kept in sync with configuration inside
+# charts/brig/templates/tests/configmap.yaml and
+# hack/bin/integration-setup-federation.sh
+export NAMESPACE=${NAMESPACE}-fed2
+
+$DIR/integration-teardown.sh