Merge pull request #1298 from wireapp/release/2020-12-21

Release 2020/12/21

.gitignore

@@ -83,4 +83,7 @@ deploy/dockerephemeral/build/smtp/
 /libs/libzauth/bzauth-c/deb/usr
 
 # Generated hie.yaml
-hie.yaml
+hie.yaml
+
+# generated files under .local
+.local

CHANGELOG.md

@@ -14,6 +14,42 @@
 
 -->
 
+# [2020-12-21]
+
+## Release Notes
+
+* upgrade spar before brig
+* upgrade nginz
+
+## Features
+
+* Increase the max allowed search results from 100 to 500. (#1282)
+
+## Bug fixes and other updates
+
+* SCIM: Allow strings for boolean values (#1296)
+* Extend SAML IdP/SCIM permissions to admins (not just owners) (#1274, #1280)
+* Clean up SCIM-invited users with expired invitation (#1264)
+* move-team: CLI to export/import team data (proof of concept, needs testing) (#1288)
+* Change some error labels for status 403 responses under `/identity-providers` (used by team-settings only) (#1274)
+* [federation] Data.Qualified: Better field names (#1290)
+* [federation] Add endpoint to get User Id by qualified handle (#1281, #1297)
+* [federation] Remove DB tables for ID mapping (#1287)
+* [federation] servantify /self endpoint, add `qualified_id` field (#1283)
+
+## Documentation
+
+* Integrate servant-swagger-ui to brig (#1270)
+
+## Internal changes
+
+* import all charts from wire-server-deploy/develop as of 2012-12-17 (#1293)
+* Migrate code for easier CI (#1294)
+* unit test and fix for null values in rendered JSON in UserProfile (#1292)
+* hscim: Bump upper bound for servant packages (#1285)
+* drive-by fix: allow federator to locally start up by specifying config (#1283)
+
+
 # 2020-12-15
 
 ## Release Notes

Makefile

@@ -1,7 +1,15 @@
-SHELL            := /usr/bin/env bash
-LANG             := en_US.UTF-8
-DOCKER_USER      ?= quay.io/wire
-DOCKER_TAG       ?= local
+SHELL                 := /usr/bin/env bash
+LANG                  := en_US.UTF-8
+DOCKER_USER           ?= quay.io/wire
+# default docker image tag is your system username, you can override it via environment variable.
+DOCKER_TAG            ?= $(USER)
+# default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
+HELM_SEMVER           ?= 0.0.42
+# The list of helm charts needed for integration tests on kubernetes
+CHARTS_INTEGRATION    := wire-server databases-ephemeral fake-aws
+# The list of helm charts to publish on S3
+# FUTUREWORK: after we "inline local subcharts", i.e. move charts/brig to charts/wire-server/brig this list could be generated from the folder names under ./charts/
+CHARTS_RELEASE        := wire-server databases-ephemeral fake-aws aws-ingress backoffice calling-test demo-smtp elasticsearch-curator elasticsearch-external fluent-bit minio-external cassandra-external nginx-ingress-controller nginx-ingress-services reaper wire-server-metrics sftd
 
 default: fast
 
@@ -198,10 +206,105 @@ db-reset:
 libzauth:
 	$(MAKE) -C libs/libzauth install
 
+#################################
 # Useful when using Haskell IDE Engine
 # https://github.com/haskell/haskell-ide-engine
 #
 # Run this again after changes to libraries or dependencies.
 .PHONY: hie.yaml
 hie.yaml:
 	stack exec gen-hie > hie.yaml
+
+#####################################
+# Today we pretend to be CI and run integration tests on kubernetes
+# (see also docs/developer/processes.md)
+#
+# NOTE: This uses local helm charts from .local/charts (which it builds before running this)
+#
+# NOTE/WARNING: By default, it uses local docker image tags,
+# which will not work at this time on your remote kubernetes cluster. [FUTUREWORK: local kubernetes cluster]
+#
+# If you wish to use docker images that are uploaded to quay.io, you must set DOCKER_TAG
+#
+#   DOCKER_TAG=<desired-wire-server-docker-tag> make kube-integration
+#
+# and if you don't know what a good DOCKER_TAG might be, you can run
+#
+#   make latest-brig-tag
+#
+# This task requires: [FUTUREWORK: add tooling setup to wire-server]
+#   - helm (version 3.1.1)
+#   - kubectl
+#   - a valid kubectl context configured (i.e. access to a kubernetes cluster)
+.PHONY: kube-integration
+kube-integration: charts
+	# by default "test-<your computer username> is used as namespace
+	export NAMESPACE=test-$(USER); ./hack/bin/integration-setup.sh
+	export NAMESPACE=test-$(USER); ./hack/bin/integration-test.sh
+
+.PHONY: kube-integration-teardown
+kube-integration-teardown:
+	export NAMESPACE=test-$(USER); ./hack/bin/integration-teardown.sh
+
+.PHONY: latest-brig-tag
+latest-brig-tag:
+	./hack/bin/find-latest-docker-tag.sh
+
+.PHONY: release-chart-%
+release-chart-%:
+	@if [ "${HELM_SEMVER}" = "0.0.42" ]; then \
+	      echo "Environment variable HELM_SEMVER not set to non-default value. Re-run with HELM_SEMVER=<something>"; \
+	    exit 1; \
+	fi
+	@if [ "${DOCKER_TAG}" = "${USER}" ]; then \
+	      echo "Environment variable DOCKER_TAG not set to non-default value. Re-run with DOCKER_TAG=<something>"; \
+	    exit 1; \
+	fi
+	make chart-$(*)
+
+
+# Rationale for copying charts to a gitignored folder before modifying helm versions and docker image tags:
+#
+# * we want to keep git history clean and not clutter it with version bump commits
+#   * synchronizing version bumps with multiple PRs, releases to master and merges back to develop is hard to do in git
+#   * we don't want to spend time modifying version tags manually all the time
+# * we want version pinning for helm charts and docker images for reproducible results during deployments
+#   * CI will keep track of versioning and upload charts to an S3 mirror
+#   * if you need to do this locally, also use this make target and set desired versions accordingly.
+.PHONY: chart-%
+chart-%:
+	./hack/bin/copy-charts.sh $(*)
+	./hack/bin/set-wire-server-image-version.sh $(DOCKER_TAG)
+	./hack/bin/set-helm-chart-version.sh "$*" $(HELM_SEMVER)
+
+# Usecase for this make target:
+#  * for local integration testing of wire-server inside kubernetes
+.PHONY: charts-integration
+charts-integration: $(foreach chartName,$(CHARTS_INTEGRATION),chart-$(chartName))
+
+# Usecase for this make target:
+# 1. for releases of helm charts
+# 2. for testing helm charts more generally
+.PHONY: charts-release
+charts-release: $(foreach chartName,$(CHARTS_RELEASE),release-chart-$(chartName))
+
+.PHONY: clean-charts
+clean-charts:
+	rm -rf .local/charts
+
+##########################################
+# Helm chart releasing (mirroring to S3)
+# Only CI should run these targets ideally
+
+# Usecases for this make target:
+# To release one single helm chart to S3 mirror
+# (assummption: CI sets DOCKER_TAG and HELM_SEMVER)
+.PHONY: upload-chart-%
+upload-chart-%: release-chart-%
+	./hack/bin/upload-helm-charts-s3.sh .local/charts/$(*)
+
+# Usecases for this make target:
+# To uplaod all helm charts in the CHARTS_RELEASE list (see top of the time)
+# (assummption: CI sets DOCKER_TAG and HELM_SEMVER)
+.PHONY: upload-charts
+upload-charts: $(foreach chartName,$(CHARTS_RELEASE),upload-chart-$(chartName))

README.md

@@ -63,6 +63,8 @@ It also contains
 - **build**: Build scripts and Dockerfiles for some platforms
 - **deploy**: (Work-in-progress) - how to run wire-server in an ephemeral, in-memory demo mode
 - **doc**: Documentation
+- **hack**: scripts and configuration for kuberentes helm chart development/releases mainly used by CI
+- **charts**: kubernetes helm charts
 
 ## Architecture Overview
 

charts/.gitignore

@@ -0,0 +1,3 @@
+# helm generated files
+*.tgz
+requirements.lock

charts/account-pages/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/account-pages/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A Helm chart for the Wire account pages in Kubernetes
+name: account-pages
+version: 0.0.42

charts/account-pages/README.md

@@ -0,0 +1 @@
+Basic web application that provides a frontend with functionality for account activation and password reset

charts/account-pages/templates/_helpers.tpl

@@ -0,0 +1,16 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "account-pages.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "account-pages.fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

charts/account-pages/templates/deployment.yaml

@@ -0,0 +1,58 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: account-pages
+  labels:
+    wireService: account-pages
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: {{ .Values.replicaCount | mul 2 }}
+  selector:
+    matchLabels:
+      wireService: account-pages
+      app: account-pages
+  template:
+    metadata:
+      labels:
+        wireService: account-pages
+        app: account-pages
+        release: {{ .Release.Name }}
+    spec:
+      containers:
+      - name: account-pages
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        env:
+          - name: BACKEND_REST
+            value: https://{{ .Values.config.externalUrls.backendRest }}
+          - name: APP_BASE
+            value: https://{{ .Values.config.externalUrls.appHost }}
+      {{- range $key, $val := .Values.envVars }}
+          - name: {{ $key }}
+            value: {{ $val | quote }}
+      {{- end }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.service.http.internalPort }}
+        readinessProbe:
+          httpGet:
+            path: /_health/
+            port: {{ .Values.service.http.internalPort }}
+            scheme: HTTP
+        livenessProbe:
+          initialDelaySeconds: 30
+          timeoutSeconds: 3
+          httpGet:
+            path: /_health/
+            port: {{ .Values.service.http.internalPort }}
+            scheme: HTTP
+        resources:
+{{ toYaml .Values.resources | indent 12 }}
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always

charts/account-pages/values.yaml

@@ -0,0 +1,49 @@
+# Default values for the account-pages.
+replicaCount: 1
+resources:
+  requests:
+    memory: "128Mi"
+    cpu: "100m"
+  limits:
+    memory: "512Mi"
+    cpu: "1"
+image:
+  repository: quay.io/wire/account
+  tag: 2.1.4-5f9c54-v0.26.5-production
+service:
+  https:
+    externalPort: 443
+  http:
+    internalPort: 8080
+
+## The following has to be provided to deploy this chart
+
+#config:
+#  externalUrls:
+#   backendRest: nginz-https.example.com
+#   backendWebsocket: nginz-ssl.example.com
+#   appHost: account.example.com
+#
+# Some relevant environment options, have a look at
+# https://github.com/wireapp/wire-account/wiki/Self-hosting
+# NOTE: Without an empty dictionary, you will see warnings
+#       when overriding envVars
+envVars: {}
+# E.g.
+# envVars:
+#   FEATURE_ENABLE_DEBUG: "true"
+#   You are likely to need at least following CSP headers
+#   due to the fact that you are likely to do cross sub-domain requests
+#   i.e., from account.example.com to nginz-https.example.com
+#   CSP_EXTRA_CONNECT_SRC: "https://*.example.com, wss://*.example.com"
+#   CSP_EXTRA_IMG_SRC: "https://*.example.com"
+#   CSP_EXTRA_SCRIPT_SRC: "https://*.example.com"
+#   CSP_EXTRA_DEFAULT_SRC: "https://*.example.com"
+#   CSP_EXTRA_FONT_SRC: "https://*.example.com"
+#   CSP_EXTRA_FRAME_SRC: "https://*.example.com"
+#   CSP_EXTRA_MANIFEST_SRC: "https://*.example.com"
+#   CSP_EXTRA_OBJECT_SRC: "https://*.example.com"
+#   CSP_EXTRA_MEDIA_SRC: "https://*.example.com"
+#   CSP_EXTRA_PREFETCH_SRC: "https://*.example.com"
+#   CSP_EXTRA_STYLE_SRC: "https://*.example.com"
+#   CSP_EXTRA_WORKER_SRC: "https://*.example.com"

charts/aws-ingress/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/aws-ingress/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: A Helm chart for ingresses (AWS specific) on Kubernetes
+name: aws-ingress
+version: 0.0.42

charts/aws-ingress/templates/ELB_account_pages_https.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.ingress.accountPages.enabled }}
+kind: Service
+apiVersion: v1
+metadata:
+  name: account-pages-elb-https
+  annotations:
+    # annotations are documented under https://kubernetes.io/docs/concepts/services-networking/service/
+    service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "{{ .Values.ingress.accountPages.https.externalPort }}"
+    service.beta.kubernetes.io/aws-load-balancer-backend-protocol: "http"
+    service.beta.kubernetes.io/aws-load-balancer-ssl-cert: "{{ .Values.ingress.accountPages.https.sslCert }}"
+    service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy: "{{ .Values.ingress.accountPages.https.sslPolicy }}"
+    external-dns.alpha.kubernetes.io/hostname:   "{{ .Values.ingress.accountPages.https.hostname }}"
+    external-dns.alpha.kubernetes.io/ttl: "{{ .Values.ingress.accountPages.https.ttl }}"
+spec:
+  type: LoadBalancer
+  selector:
+    wireService: account-pages
+  ports:
+  - name: https
+    protocol: TCP
+    port: {{ .Values.ingress.accountPages.https.externalPort }}
+    # NOTE: This value should match team settings http listening port
+    targetPort: {{ .Values.ingress.accountPages.http.accountPagesPort }}
+{{- end }}