Merge pull request #596 from wireapp/release/2019-01-24

Release/2019-01-24

.dockerignore

@@ -1,2 +1,5 @@
 **/.*
 **/dist
+**/target
+**/*.aci
+services/nginz/src/objs

.gitignore

@@ -16,6 +16,8 @@ Cargo.lock
 *.asc
 *.tmp
 *~
+.#*
+*#*#
 .*.sw[a-z]
 .cabal-sandbox
 ID
@@ -60,3 +62,5 @@ integration-aws.yaml
 DOCKER_ID*
 swagger-ui
 services/spar/spar.cabal
+deploy/services-demo/resources/templates/*
+deploy/services-demo/conf/nginz/zwagger-ui/*

CHANGELOG.md

@@ -1,3 +1,30 @@
+# 2019-01-27 #596
+
+## API changes
+
+  * Track inviters of team members (#566)
+  * New partner role. (#569, #572, #573, #576, #579, #584, #577, #592)
+  * App-level websocket pongs. (#561)
+
+## Bug fixes
+
+  * Spar re-login deleted sso users; fix handling of brig errors. (#588)
+  * Gundeck: lost push notifications with push-all enabled.  (#554)
+  * Gundeck: do not push natively to devices if they are not on the whitelist.  (#554)
+  * Gundeck: link gundeck unit tests with -threaded.  (#554)
+
+## Internal changes
+
+  * Get rid of async-pool (unliftio now provides the same functionality) (#568)
+  * Fix: log multi-line error messages on one line. (#595)
+  * Whitelist all wire.com email addresses (#578)
+  * SCIM -> Scim (#581)
+  * Changes to make the demo runnable from Docker (#571)
+  * Feature/docker image consistency (#570)
+  * add a readme, for how to build libzauth. (#591)
+  * better support debian style machines of different architecturs (#582, #587, #583, #585, #590, #580)
+
+
 # 2019-01-10 #567
 
 ## API changes

Makefile

@@ -1,6 +1,6 @@
 SHELL            := /usr/bin/env bash
 LANG             := en_US.UTF-8
-DOCKER_USER      ?= wireserver
+DOCKER_USER      ?= quay.io/wire
 DOCKER_TAG       ?= local
 
 default: fast
@@ -89,33 +89,51 @@ i-%:
 
 .PHONY: docker-deps
 docker-deps:
+	# `docker-deps` needs to be built or pulled only once (unless native dependencies change)
 	$(MAKE) -C build/alpine deps
 
 .PHONY: docker-builder
 docker-builder:
+	# `docker-builder` needs to be built or pulled only once (unless native dependencies change)
 	$(MAKE) -C build/alpine builder
 
 .PHONY: docker-intermediate
 docker-intermediate:
-	docker build -t $(DOCKER_USER)/intermediate:$(DOCKER_TAG) -f build/alpine/Dockerfile.intermediate .;
-	docker tag $(DOCKER_USER)/intermediate:$(DOCKER_TAG) $(DOCKER_USER)/intermediate:latest;
-	if test -n "$$DOCKER_PUSH"; then docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD); docker push $(DOCKER_USER)/intermediate:$(DOCKER_TAG); docker push $(DOCKER_USER)/intermediate:latest; fi;
+	# `docker-intermediate` needs to be built whenever code changes - this essentially runs `stack clean && stack install` on the whole repo
+	docker build -t $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG) -f build/alpine/Dockerfile.intermediate --build-arg intermediate=$(DOCKER_USER)/alpine-intermediate --build-arg deps=$(DOCKER_USER)/alpine-deps .;
+	docker tag $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG) $(DOCKER_USER)/alpine-intermediate:latest;
+	if test -n "$$DOCKER_PUSH"; then docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD); docker push $(DOCKER_USER)/alpine-intermediate:$(DOCKER_TAG); docker push $(DOCKER_USER)/alpine-intermediate:latest; fi;
 
 .PHONY: docker-migrations
 docker-migrations:
-	docker build -t $(DOCKER_USER)/migrations:$(DOCKER_TAG) -f build/alpine/Dockerfile.migrations .
+	# `docker-migrations` needs to be built whenever docker-intermediate was rebuilt AND new schema migrations were added.
+	docker build -t $(DOCKER_USER)/migrations:$(DOCKER_TAG) -f build/alpine/Dockerfile.migrations --build-arg intermediate=$(DOCKER_USER)/alpine-intermediate --build-arg deps=$(DOCKER_USER)/alpine-deps .
 	docker tag $(DOCKER_USER)/migrations:$(DOCKER_TAG) $(DOCKER_USER)/migrations:latest
 	if test -n "$$DOCKER_PUSH"; then docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD); docker push $(DOCKER_USER)/migrations:$(DOCKER_TAG); docker push $(DOCKER_USER)/migrations:latest; fi;
 
 .PHONY: docker-exe-%
 docker-exe-%:
-	docker build -t $(DOCKER_USER)/"$*":$(DOCKER_TAG) -f build/alpine/Dockerfile.executable --build-arg executable="$*" .
+	docker image ls | grep $(DOCKER_USER)/alpine-deps > /dev/null || (echo "'make docker-deps' required.", exit 1)
+	docker image ls | grep $(DOCKER_USER)/alpine-intermediate > /dev/null || (echo "'make docker-intermediate' required."; exit 1)
+	docker build -t $(DOCKER_USER)/"$*":$(DOCKER_TAG) -f build/alpine/Dockerfile.executable --build-arg executable="$*" --build-arg intermediate=$(DOCKER_USER)/alpine-intermediate --build-arg deps=$(DOCKER_USER)/alpine-deps .
 	docker tag $(DOCKER_USER)/"$*":$(DOCKER_TAG) $(DOCKER_USER)/"$*":latest
 	if test -n "$$DOCKER_PUSH"; then docker login -u $(DOCKER_USERNAME) -p $(DOCKER_PASSWORD); docker push $(DOCKER_USER)/"$*":$(DOCKER_TAG); docker push $(DOCKER_USER)/"$*":latest; fi;
 
-.PHONY: docker-service-%
-docker-service-%:
-	$(MAKE) -C "services/$*" docker
+.PHONY: docker-services
+docker-services:
+	# make docker-services doesn't compile, only makes small images out of the `docker-intermediate` image
+	# to recompile, run `docker-intermediate` first.
+	docker image ls | grep $(DOCKER_USER)/alpine-deps > /dev/null || (echo "'make docker-deps' required.", exit 1)
+	docker image ls | grep $(DOCKER_USER)/alpine-intermediate > /dev/null || (echo "'make docker-intermediate' required."; exit 1)
+	# `make -C services/brig docker` == `make docker-exe-brig docker-exe-brig-integration docker-exe-brig-schema docker-exe-brig-index`
+	$(MAKE) -C services/brig docker
+	$(MAKE) -C services/gundeck docker
+	$(MAKE) -C services/galley docker
+	$(MAKE) -C services/cannon docker
+	$(MAKE) -C services/proxy docker
+	$(MAKE) -C services/spar docker
+	$(MAKE) docker-exe-zauth
+	$(MAKE) -C services/nginz docker
 
 DOCKER_DEV_NETWORK := --net=host
 DOCKER_DEV_VOLUMES := -v `pwd`:/src/wire-server

README.md

@@ -29,6 +29,7 @@ See more in "[Open sourcing Wire server code](https://medium.com/@wireapp/open-s
         * [1. Compile sources natively.](#1-compile-sources-natively)
         * [2. Use docker](#2-use-docker)
     * [How to run integration tests](#how-to-run-integration-tests)
+    * [when you need more fine-grained control over your build-test loops](#when-you-need-more-fine-grained-control-over-your-build-test-loops)
 * [How to install and run `wire-server`](#how-to-install-and-run-wire-server)
 
 <!-- vim-markdown-toc -->
@@ -109,7 +110,12 @@ For building nginz, see [services/nginz/README.md](services/nginz/README.md)
 If you wish to build your own docker images, you need [docker version >= 17.05](https://www.docker.com/) and [`make`](https://www.gnu.org/software/make/). Then,
 
 ```bash
-make docker-services
+# optionally:
+# make docker-builder # if you don't run this, it pulls the alpine-builder image from quay.io
+make docker-deps docker-intermediate docker-services
+
+# subsequent times, after changing code, if you wish to re-create docker images, it's sufficient to
+make docker-intermediate docker-services
 ```
 
 will, eventually, have built a range of docker images. Make sure to [give Docker enough RAM](https://github.com/wireapp/wire-server/issues/562); if you see `make: *** [builder] Error 137`, it might be a sign that the build ran out of memory. You can also mix and match – e.g. pull the [`alpine-builder`](https://quay.io/repository/wire/alpine-builder?tab=tags) image and build the rest locally.

build/alpine/Dockerfile.executable

@@ -6,7 +6,7 @@
 #   (from wire-server root directory)
 #   export EXECUTABLE=galley-schema; docker build -t $EXECUTABLE -f build/alpine/Dockerfile.executable --build-arg executable=$EXECUTABLE .
 
-ARG intermediate=quay.io/wire/intermediate
+ARG intermediate=quay.io/wire/alpine-intermediate
 ARG deps=quay.io/wire/alpine-deps
 
 #--- Intermediate stage ---

build/alpine/README.md

@@ -7,24 +7,27 @@ To create docker images, you need to install [docker version >= 17.05](https://w
 
 Both of the above need to be built first (only once) to be able to actually build a service docker image.
 
-* `Dockerfile` depends on the two images above to compile code inside the `builder` image and then copy the resulting binary into the `deps` image to finally construct a service (e.g. `brig`) docker image that can be used for testing or deployment. The final image is ~30MB (compressed) in size.
+* `Dockerfile.intermediate` - based on `Dockerfile.deps`/`Dockerfile.builder`, this is an intermediate image compiling all dynamically linked binaries (obtained when running `make install` in the top-level directory).
+* `Dockerfile.executable` - based on `Dockerfile.deps`/`Dockerfile.intermediate`, this extracts a single executable from the intermediate image, yielding a small image (~30MB compressed) with a single dynamically linked binary.
+
 
 ### Build the `builder` and `deps` docker images locally
 
+(from within the `wire-server` directory)
 ```bash
-cd build/alpine && make
+make docker-builder
+make docker-deps
 ```
 
-### Build a service, e.g. brig:
+### Build a service docker image, e.g. brig:
 
 ```bash
-cd services/brig && make docker
+make docker-intermediate # recompiles all the haskell code
+make docker-exe-brig # this only extracts one binary from the intermediate image above and makes it the default entrypoint. Nothing gets recompiled
 ```
 
 ## Other dockerfiles
 
-* `Dockerfile.intermediate` - based on `Dockerfile.deps`/`Dockerfile.builder`, this is an intermediate image compiling all dynamically linked binaries (obtained when running `make install` in the top-level directory).
-* `Dockerfile.executable` - based on `Dockerfile.deps`/`Dockerfile.intermediate`, this extracts a single executable from the intermediate image, yielding a small image with a single dynamically linked binary.
 * `Dockerfile.migrations` - same as `Dockerfile.executable`, with a fixed set of database migration binaries.
 * `Dockerfile.prebuilder` - dependencies of `Dockerfile.builder` that are expected to change very rarely (GHC, system libraries). Currently we're able to use system GHC, but if we require a newer version of GHC than the one provided by Alpine, we could build GHC in `Dockerfile.prebuilder` (as it has been [done before][2018-11-28]).
 

deploy/docker-ephemeral/docker-compose.yaml

@@ -1,16 +1,27 @@
 version: '2'
+networks:
+  demo_wire:
+    external: false
+
 services:
   fake_dynamodb:
+    container_name: demo_wire_dynamodb
     image: cnadiminti/dynamodb-local:2018-04-11
     ports:
       - 127.0.0.1:4567:8000
+    networks:
+      - demo_wire
 
   fake_sqs:
+    container_name: demo_wire_sqs
     image: airdock/fake-sqs:0.3.1
     ports:
       - 127.0.0.1:4568:4568
+    networks:
+      - demo_wire
 
   fake_localstack:
+    container_name: demo_wire_localstack
     image: localstack/localstack:0.8.0  # NB: this is younger than 0.8.6!
     ports:
       - 127.0.0.1:4569:4579 # ses # needed for local integration tests
@@ -19,44 +30,61 @@ services:
       - DEBUG=1
       - DEFAULT_REGION=eu-west-1
       - SERVICES=ses,sns
+    networks:
+      - demo_wire
 
   basic_smtp: # needed for demo setup
+    container_name: demo_wire_smtp
     # https://github.com/namshi/docker-smtp
     image: namshi/smtp
     ports:
         - 127.0.0.1:2500:25
+    networks:
+      - demo_wire
 
   fake_s3:
+    container_name: demo_wire_s3
     image: minio/minio:RELEASE.2018-05-25T19-49-13Z
     ports:
      - "127.0.0.1:4570:9000"
     environment:
       MINIO_ACCESS_KEY: dummykey
       MINIO_SECRET_KEY: dummysecret # minio requires a secret of at least 8 chars
     command: server /tmp
+    networks:
+      - demo_wire
 
   # activemq:
   #   image: rmohr/activemq:5.15.4
   #   ports:
   #     - "61613:61613"
 
   redis:
+    container_name: demo_wire_redis
     image: redis:3.0.7-alpine
     ports:
       - "127.0.0.1:6379:6379"
+    networks:
+      - demo_wire
 
   elasticsearch:
+    container_name: demo_wire_elasticsearch
     image: elasticsearch:5.6
     # https://hub.docker.com/_/elastic is deprecated, but 6.2.4 did not work without further changes.
     # image: docker.elastic.co/elasticsearch/elasticsearch:6.2.4
     ports:
       - "127.0.0.1:9200:9200"
       - "127.0.0.1:9300:9300"
+    networks:
+      - demo_wire
 
   cassandra:
+    container_name: demo_wire_cassandra
     image: cassandra:3.11.2
     ports:
       - "127.0.0.1:9042:9042"
+    networks:
+      - demo_wire
 
   db_migrations_brig_schema:
     image: quay.io/wire/brig-schema
@@ -67,6 +95,8 @@ services:
       - ./:/scripts
     links:
       - cassandra
+    networks:
+      - demo_wire
 
   db_migrations_brig_index:
     image: quay.io/wire/brig-index
@@ -77,6 +107,8 @@ services:
       - ./:/scripts
     links:
       - elasticsearch
+    networks:
+      - demo_wire
 
   db_migrations_galley:
     image: quay.io/wire/galley-schema
@@ -87,6 +119,8 @@ services:
       - ./:/scripts
     links:
       - cassandra
+    networks:
+      - demo_wire
 
   db_migrations_gundeck:
     image: quay.io/wire/gundeck-schema
@@ -97,6 +131,8 @@ services:
       - ./:/scripts
     links:
       - cassandra
+    networks:
+      - demo_wire
 
   db_migrations_spar:
     image: quay.io/wire/spar-schema
@@ -107,6 +143,8 @@ services:
       - ./:/scripts
     links:
       - cassandra
+    networks:
+      - demo_wire
 
   aws_cli:
     image: mesosphere/aws-cli:1.14.5
@@ -124,3 +162,5 @@ services:
     entrypoint: /scripts/init.sh
     volumes:
       - ./:/scripts
+    networks:
+      - demo_wire

deploy/services-demo/README.md

@@ -12,7 +12,8 @@ deploy/docker-ephemeral/run.sh
 
 ```
 # On terminal 2, start the services
-deploy/services-demo/demo.sh
+deploy/services-demo/demo.sh  # if all services have been compiled natively
+deploy/services-demo/demo.sh docker # in case Docker images were built instead
 ```
 
 ### Structure of the services-demo folder