Merge pull request #2664 from wireapp/release_2022-09-01_09_47

Release 2022-09-01 - (expected chart version 4.23.0)

.github/pull_request_template.md

@@ -1,15 +1,5 @@
 ## Checklist
 
- - [ ] The **PR Title** explains the impact of the change.
- - [ ] The **PR description** provides context as to why the change should occur and what the code contributes to that effect. This could also be a link to a JIRA ticket or a Github issue, if there is one.
- - [ ] If this PR changes development workflow or dependencies, they have been A) automated and B) documented under docs/developer/. All efforts have been taken to minimize development setup breakage or slowdown for co-workers.
- - [ ] If HTTP endpoint paths have been added or renamed, or feature configs have changed, the **endpoint / config-flag checklist** (see Wire-employee only backend [wiki page](https://github.com/zinfra/backend-wiki/wiki/Checklists)) has been followed.
- - [ ] If a cassandra schema migration has been added, I ran **`make git-add-cassandra-schema`** to update the cassandra schema documentation.
- - [ ] **changelog.d** contains the following bits of information ([details](https://github.com/wireapp/wire-server/blob/develop/docs/developer/changelog.md)):
-   - [ ] A file with the changelog entry in one or more suitable sub-sections. The sub-sections are marked by directories inside `changelog.d`.
-   - [ ] If new config options introduced: added usage description under docs/reference/config-options.md
-   - [ ] If new config options introduced: recommended measures to be taken by on-premise instance operators.
-   - [ ] If a cassandra schema migration is backwards incompatible (see also [these docs](https://github.com/wireapp/wire-server/blob/develop/docs/developer/cassandra-interaction.md#cassandra-schema-migrations)), measures to be taken by on-premise instance operators are explained.
-   - [ ] If a data migration (not schema migration) introduced: measures to be taken by on-premise instance operators.
-   - [ ] If public end-points have been changed or added: does nginz need un upgrade?
-   - [ ] If internal end-points have been added or changed: which services have to be deployed in a specific order?
+ - [ ] Add a new entry in an appropriate subdirectory of `changelog.d`
+ - [ ] Read and follow the
+[PR guidelines](https://github.com/wireapp/wire-server/blob/develop/docs/developer/pr-guidelines.md)

.gitignore

@@ -82,8 +82,6 @@ deploy/dockerephemeral/build/smtp/
 /libs/libzauth/bzauth-c/deb/usr
 
 # Generated by "make hie.yaml"
-hie.yaml
-hie.orig.yaml
 stack-dev.yaml
 
 # HIE db files (e.g. generated for stan)

CHANGELOG.md

@@ -1,3 +1,95 @@
+# [2022-09-01] (Chart Release 4.23.0)
+
+## Release notes
+
+
+* The internal endpoint `GET i/mls/clients` has been changed, and it now returns a list of `ClientInfo` instead of a list of `ClientId`. (#2631)
+
+
+## API changes
+
+
+* Fix key package error description (#2651)
+
+* Expose MLS public keys in a new endpoint `GET /mls/public-keys`. (#2602)
+
+
+## Features
+
+
+* The coturn chart now supports exposing the control port over TLS. (#2620)
+
+* Forward all MLS default proposal types (#2628)
+
+* New endpoints `HEAD` and `GET /nonce/clients` to request new nonces for client certificate requests (coming up soon). (#2641, #2655)
+
+## Bug fixes and other updates
+
+
+* Fix cql-io bug where restarting whole cassandra cluster could cause downtime. Upstream changes in https://gitlab.com/twittner/cql-io/-/merge_requests/20 (#2640)
+
+* Improve client check when adding clients to MLS conversations (#2631)
+
+
+## Documentation
+
+
+* Move developer docs onto docs.wire.com (instead of exposing them on github only) (#2622, #2649)
+
+* Add build instructions for developers (#2621)
+
+* Make target audience explicit on docs.wire.com (#2662)
+
+
+## Internal changes
+
+
+* Support for external Add proposals (#2567)
+
+* Add additional checks on incoming MLS messages:
+  * if the sender matches the authenticated user
+  * if the sender of message to a remote conversation is a member
+  * if the group ID of a remote conversation matches the local mapping (#2618)
+
+* Apply changes introduced by cabal-fmt. (#2624)
+
+* Remove some redudant constraints in brig (#2638)
+
+* Brig Polysemy: Port UserPendingActivationStore to polysemy (#2636)
+
+
+* Add make target `delete-cache-on-linker-errors` to delete all Haskell compilation related caches. This is useful in cases where the development environment gets into an inconsistent state. (#2623)
+
+
+* Move Paging effect from galley into polysemy-wire-zoo (#2648)
+
+* Fix broken hls-hlint-plugin in nix env (#2629)
+
+* Adjust developer PR template and document config and API procedures in-tree. (#2617)
+
+* Add mls-test-cli to builder image (#2626)
+
+* Add mls-test-cli to deps image (#2630)
+
+* mls-test-cli: Use Cargo.lock file when building (#2634)
+
+* Move common Arbitrary instances to types-common package for compilation speed (#2658)
+
+* `LoginId` migrated to schema-profunctor (#2633, #2645)
+
+* Improve cleaning rules in Makefile. (#2639)
+
+* Fix typos, dangling reference in source code haddocs, etc. (#2586)
+
+* Update the Elastic Search version used for running integration tests to the one that is delivered by wire-server-deploy. (#2656)
+
+
+## Federation changes
+
+
+* Add mlsPrivateKeyPaths setting to galley (#2602)
+
+
 # [2022-08-16] (Chart Release 4.22.0)
 
 ## API changes

Makefile

@@ -50,6 +50,38 @@ else
 	stack install --pedantic --test --bench --no-run-benchmarks --local-bin-path=dist
 endif
 
+# Clean
+.PHONY: full-clean
+full-clean: clean
+	rm -rf ~/.cache/hie-bios
+ifdef CABAL_DIR
+	rm -rf $(CABAL_DIR)/store
+else
+	rm -rf ~/.cabal/store
+endif
+
+.PHONY: clean
+clean:
+ifeq ($(WIRE_BUILD_WITH_CABAL), 1)
+	cabal clean
+else
+	stack clean
+endif
+	$(MAKE) -C services/nginz clean
+	-rm -rf dist
+	-rm -f .metadata
+
+.PHONY: clean-hint
+clean-hint:
+	@echo -e "\n\n\n>>> PSA: if you get errors that are hard to explain,"
+	@echo -e ">>> try 'make full-clean' and run your command again."
+	@echo -e ">>> see https://github.com/wireapp/wire-server/blob/develop/docs/developer/building.md#linker-errors-while-compiling\n\n\n"
+
+.PHONY: cabal.project.local
+cabal.project.local:
+	echo "optimization: False" > ./cabal.project.local
+	./hack/bin/cabal-project-local-template.sh "ghc-options: -O0" >> ./cabal.project.local
+
 # Build all Haskell services and executables with -O0, run unit tests
 .PHONY: fast
 fast: init
@@ -62,7 +94,7 @@ endif
 # Usage: make c package=brig test=1
 .PHONY: c
 c: cabal-fmt
-	cabal build $(WIRE_CABAL_BUILD_OPTIONS) $(package)
+	cabal build $(WIRE_CABAL_BUILD_OPTIONS) $(package) || ( make clean-hint; false )
 ifeq ($(test), 1)
 	./hack/bin/cabal-run-tests.sh $(package) $(testargs)
 endif
@@ -141,18 +173,6 @@ add-license:
 shellcheck:
 	./hack/bin/shellcheck.sh
 
-# Clean
-.PHONY: clean
-clean:
-ifeq ($(WIRE_BUILD_WITH_CABAL), 1)
-	cabal clean
-else
-	stack clean
-endif
-	$(MAKE) -C services/nginz clean
-	-rm -rf dist
-	-rm -f .metadata
-
 #################################
 ## running integration tests
 

README.md

@@ -18,22 +18,6 @@ For documentation on how to self host your own Wire-Server see [this section](#h
 
 See more in "[Open sourcing Wire server code](https://medium.com/@wireapp/open-sourcing-wire-server-code-ef7866a731d5)".
 
-## Table of contents
-
-<!-- vim-markdown-toc GFM -->
-
-* [Contents of this repository](#contents-of-this-repository)
-* [Architecture Overview](#architecture-overview)
-* [Development setup](#development-setup)
-    * [How to build `wire-server` binaries](#how-to-build-wire-server-binaries)
-        * [1. Compile sources natively.](#1-compile-sources-natively)
-        * [2. Use docker](#2-use-docker)
-    * [How to run integration tests](#how-to-run-integration-tests)
-    * [when you need more fine-grained control over your build-test loops](#when-you-need-more-fine-grained-control-over-your-build-test-loops)
-* [How to install and run `wire-server`](#how-to-install-and-run-wire-server)
-
-<!-- vim-markdown-toc -->
-
 ## Contents of this repository
 
 This repository contains the following source code:
@@ -82,29 +66,7 @@ private network.
 
 There are two options:
 
-#### 1. Compile sources natively.
-
-This requires a range of dependencies that depend on your platform/OS, such as:
-
-- Haskell & Rust compiler and package managers
-- Some package dependencies (libsodium, openssl, protobuf, icu, geoip, snappy, [cryptobox-c](https://github.com/wireapp/cryptobox-c), ...) that depend on your platform/OS
-
-See [docs/developer/dependencies.md](docs/legacy/developer/dependencies.md) for details.
-
-Once all dependencies are set up, the following should succeed:
-
-```bash
-# build all haskell services
-make
-# build one haskell service, e.g. brig:
-cd services/brig && make
-```
-
-The default make target (`fast`) compiles unoptimized (faster compilation time, slower binaries), which should be fine for development purposes. Use `make install` to get optimized binaries.
-
-For building nginz, see [services/nginz/README.md](services/nginz/README.md)
-
-#### 2. Use docker
+#### 1. Use docker
 
 *If you don't wish to build all docker images from scratch (e.g. the `ubuntu20-builder` takes a very long time), ready-built images can be downloaded from [here](https://quay.io/organization/wire).*
 
@@ -123,54 +85,9 @@ will, eventually, have built a range of docker images. Make sure to [give Docker
 
 See the `Makefile`s and `Dockerfile`s, as well as [build/ubuntu/README.md](build/ubuntu/README.md) for details.
 
-### How to run integration tests
-
-Integration tests require all of the haskell services (brig, galley, cannon, gundeck, proxy, cargohold, spar) to be correctly configured and running, before being able to execute e.g. the `brig-integration` binary. The test for brig also starts nginz, so make sure it has been built before.
-These services require most of the deployment dependencies as seen in the architecture diagram to also be available:
-
-- Required internal dependencies:
-    - cassandra (with the correct schema)
-    - elasticsearch (with the correct schema)
-    - redis
-- Required external dependencies are the following configured AWS services (or "fake" replacements providing the same API):
-    - SES
-    - SQS
-    - SNS
-    - S3
-    - DynamoDB
-- Required additional software:
-    - netcat (in order to allow the services being tested to talk to the dependencies above)
-
-Setting up these real, but in-memory internal and "fake" external dependencies is done easiest using [`docker-compose`](https://docs.docker.com/compose/install/). Run the following in a separate terminal (it will block that terminal, C-c to shut all these docker images down again):
-
-```
-deploy/dockerephemeral/run.sh
-```
-
-Then, to run all integration tests:
-
-```bash
-make integration
-```
-
-Or, alternatively, `make` on the top-level directory (to produce all the service's binaries) followed by e.g `cd services/brig && make integration` to run one service's integration tests only.
-
-### when you need more fine-grained control over your build-test loops
-
-You can use `$WIRE_STACK_OPTIONS` to pass arguments to stack through the `Makefile`s.  This is useful to e.g. pass arguments to a unit test suite or temporarily disable `-Werror` without the risk of accidentally committing anything, like this:
-
-```bash
-WIRE_STACK_OPTIONS='--ghc-options=-Wwarn --test-arguments="--quickcheck-tests=19919 --quickcheck-replay=651712"' make -C services/gundeck
-```
-
-Integration tests are run via `/services/integration.sh`, which does not know about stack or `$WIRE_STACK_OPTIONS`.  Here you can use `$WIRE_INTEGRATION_TEST_OPTIONS`:
-
-```bash
-cd services/spar
-WIRE_INTEGRATION_TEST_OPTIONS="--match='POST /identity-providers'" make i
-```
+#### 2. Use nix-provided build environment 
 
-Alternatively, you can use [tasty's support for passing arguments vie shell variables directly](https://github.com/feuerbach/tasty#runtime).  Or, in the case of spar, the [hspec equivalent](https://hspec.github.io/options.html#specifying-options-through-an-environment-variable), which [is less helpful at times](https://github.com/hspec/hspec/issues/335).
+This is suitable only for local development and testing. See [build instructions](./docs/developer/building.md) in the developer documentation.
 
 ## How to install and run `wire-server`
 

build/ubuntu/Dockerfile.builder

@@ -1,6 +1,19 @@
 ARG prebuilder=quay.io/wire/ubuntu20-prebuilder
 
+FROM rust:1.63 as mls-test-cli-builder
+
+# compile mls-test-cli tool
+RUN cd /tmp && \
+    git clone https://github.com/wireapp/mls-test-cli && \
+    cd mls-test-cli && \
+    git rev-parse HEAD
+
+RUN cd /tmp/mls-test-cli && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu
+
 FROM ${prebuilder}
+
+COPY --from=mls-test-cli-builder /tmp/mls-test-cli/target/x86_64-unknown-linux-gnu/release/mls-test-cli /usr/bin/mls-test-cli
+
 WORKDIR /
 
 # Download stack indices and compile/cache dependencies to speed up subsequent

build/ubuntu/Dockerfile.deps

@@ -1,12 +1,3 @@
-FROM rust:1.63 as mls-test-cli-builder
-
-# compile mls-test-cli tool
-RUN cd /tmp && \
-    git clone https://github.com/wireapp/mls-test-cli && \
-    cd mls-test-cli && \
-    cargo build --release
-
-
 FROM ubuntu:20.04 as cryptobox-builder
 
 # compile cryptobox-c
@@ -19,15 +10,21 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     export SODIUM_USE_PKG_CONFIG=1 && \
     cargo build --release
 
+FROM rust:1.63 as mls-test-cli-builder
+
+# compile mls-test-cli tool
+RUN cd /tmp && \
+    git clone https://github.com/wireapp/mls-test-cli && \
+    cd mls-test-cli && \
+    git rev-parse HEAD
+
+RUN cd /tmp/mls-test-cli && RUSTFLAGS='-C target-feature=+crt-static' cargo build --release --target x86_64-unknown-linux-gnu
 
 # Minimal dependencies for ubuntu-compiled, dynamically linked wire-server Haskell services
 FROM ubuntu:20.04
 
 COPY --from=cryptobox-builder /tmp/cryptobox-c/target/release/libcryptobox.so /usr/lib
-
-# FUTUREWORK: only copy mls-test-cli executables if we are building an
-# integration test image
-COPY --from=mls-test-cli-builder /tmp/mls-test-cli/target/release/mls-test-cli /usr/bin
+COPY --from=mls-test-cli-builder /tmp/mls-test-cli/target/x86_64-unknown-linux-gnu/release/mls-test-cli /usr/bin/mls-test-cli
 
 RUN export DEBIAN_FRONTEND=noninteractive && \
     apt-get update && \

cabal.project

@@ -142,6 +142,11 @@ source-repository-package
     location: https://github.com/wireapp/saml2-web-sso
     tag: 74371cd775cb98d6cf85f6e182244a3c4fd48702
 
+source-repository-package
+    type: git
+    location: https://gitlab.com/axeman/cql-io
+    tag: c2b6aa995b5817ed7c78c53f72d5aa586ef87c36
+
 source-repository-package
     type: git
     location: https://gitlab.com/axeman/swagger

cabal.project.freeze

@@ -575,7 +575,6 @@ constraints: any.AC-Angle ==1.0,
              any.cpu ==0.1.2,
              any.cpuinfo ==0.1.0.2,
              any.cql ==4.0.3,
-             any.cql-io ==1.1.1,
              any.cql-io-tinylog ==0.1.0,
              any.crackNum ==3.1,
              any.crc32c ==0.0.0,
@@ -1153,6 +1152,8 @@ constraints: any.AC-Angle ==1.0,
              any.hourglass ==0.2.12,
              any.hourglass-orphans ==0.1.0.0,
              any.hp2pretty ==0.10,
+             any.hpack ==0.34.5,
+             any.hpack-dhall ==0.5.3,
              any.hpc-codecov ==0.3.0.0,
              any.hpc-lcov ==1.0.1,
              any.hprotoc ==2.4.17,
@@ -2457,6 +2458,7 @@ constraints: any.AC-Angle ==1.0,
              any.time-units ==1.0.0,
              any.timeit ==2.0,
              any.timelens ==0.2.0.2,
+             any.timeout ==0.1.1,
              any.timer-wheel ==0.3.0,
              any.timerep ==2.0.1.0,
              any.timezone-olson ==0.2.0,